go_study/fabric-main/orderer/common/server/etcdraft_test.go

335 lines
16 KiB
Go

/*
Copyright IBM Corp. All Rights Reserved.
SPDX-License-Identifier: Apache-2.0
*/
package server_test
import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"sync/atomic"
"testing"
"time"
. "github.com/onsi/gomega"
"github.com/onsi/gomega/gbytes"
"github.com/onsi/gomega/gexec"
)
var basePort = int32(8000)
func nextPort() int32 {
return atomic.AddInt32(&basePort, 1)
}
func TestSpawnEtcdRaft(t *testing.T) {
gt := NewGomegaWithT(t)
// Build the configtxgen binary
configtxgen, err := gexec.Build("github.com/hyperledger/fabric/cmd/configtxgen")
gt.Expect(err).NotTo(HaveOccurred())
cryptogen, err := gexec.Build("github.com/hyperledger/fabric/cmd/cryptogen")
gt.Expect(err).NotTo(HaveOccurred())
// Build the orderer binary
orderer, err := gexec.Build("github.com/hyperledger/fabric/cmd/orderer")
gt.Expect(err).NotTo(HaveOccurred())
defer gexec.CleanupBuildArtifacts()
tempSharedDir := t.TempDir()
copyYamlFiles(gt, "testdata", tempSharedDir)
cryptoPath := generateCryptoMaterials(gt, cryptogen, tempSharedDir)
t.Run("Bad", func(t *testing.T) {
t.Run("Invalid bootstrap method", func(t *testing.T) {
testEtcdRaftOSNFailureInvalidBootstrapMethod(NewGomegaWithT(t), tempSharedDir, orderer, configtxgen, cryptoPath)
})
t.Run("TLS disabled single listener", func(t *testing.T) {
testEtcdRaftOSNNoTLSSingleListener(NewGomegaWithT(t), tempSharedDir, orderer, configtxgen, cryptoPath)
})
})
t.Run("Good", func(t *testing.T) {
// tests in this suite actually launch process with success, hence we need to avoid
// conflicts in listening port, opening files.
t.Run("TLS disabled dual listener", func(t *testing.T) {
testEtcdRaftOSNNoTLSDualListener(NewGomegaWithT(t), tempSharedDir, orderer, configtxgen, cryptoPath)
})
t.Run("TLS enabled single listener", func(t *testing.T) {
testEtcdRaftOSNSuccess(NewGomegaWithT(t), tempSharedDir, configtxgen, orderer, cryptoPath)
})
t.Run("Start orderer without channels", func(t *testing.T) {
testEtcdRaftOSNStart(NewGomegaWithT(t), tempSharedDir, configtxgen, orderer, cryptoPath)
})
t.Run("Restart orderer after joining application channel", func(t *testing.T) {
testEtcdRaftOSNJoinAppChan(NewGomegaWithT(t), tempSharedDir, configtxgen, orderer, cryptoPath)
})
t.Run("Channel participation disabled is ignored", func(t *testing.T) {
testEtcdRaftOSNFailureChannelParticipationDisabled(NewGomegaWithT(t), tempSharedDir, orderer, configtxgen, cryptoPath)
})
})
}
func copyYamlFiles(gt *GomegaWithT, src, dst string) {
for _, file := range []string{"configtx.yaml", "examplecom-config.yaml", "orderer.yaml"} {
fileBytes, err := ioutil.ReadFile(filepath.Join(src, file))
gt.Expect(err).NotTo(HaveOccurred())
err = ioutil.WriteFile(filepath.Join(dst, file), fileBytes, 0o644)
gt.Expect(err).NotTo(HaveOccurred())
}
}
func generateBootstrapBlock(gt *GomegaWithT, tempDir, configtxgen, channel, profile string) string {
// create a genesis block for the specified channel and profile
genesisBlockPath := filepath.Join(tempDir, "genesis.block")
cmd := exec.Command(
configtxgen,
"-channelID", channel,
"-profile", profile,
"-outputBlock", genesisBlockPath,
"--configPath", tempDir,
)
configtxgenProcess, err := gexec.Start(cmd, nil, nil)
gt.Expect(err).NotTo(HaveOccurred())
gt.Eventually(configtxgenProcess, time.Minute).Should(gexec.Exit(0))
gt.Expect(configtxgenProcess.Err).To(gbytes.Say("Writing genesis block"))
return genesisBlockPath
}
func generateCryptoMaterials(gt *GomegaWithT, cryptogen, path string) string {
cryptoPath := filepath.Join(path, "crypto")
cmd := exec.Command(
cryptogen,
"generate",
"--config", filepath.Join(path, "examplecom-config.yaml"),
"--output", cryptoPath,
)
cryptogenProcess, err := gexec.Start(cmd, nil, nil)
gt.Expect(err).NotTo(HaveOccurred())
gt.Eventually(cryptogenProcess, time.Minute).Should(gexec.Exit(0))
return cryptoPath
}
func testEtcdRaftOSNStart(gt *GomegaWithT, tempDir, configtxgen, orderer, cryptoPath string) {
// Launch the OSN
ordererProcess := launchOrderer(gt, orderer, tempDir, tempDir, "", cryptoPath, "none", "true", "info")
defer func() { gt.Eventually(ordererProcess.Kill(), time.Minute).Should(gexec.Exit()) }()
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("Beginning to serve requests"))
}
func testEtcdRaftOSNJoinAppChan(gt *GomegaWithT, configPath, configtxgen, orderer, cryptoPath string) {
tempDir, err := ioutil.TempDir("", "etcdraft-test")
gt.Expect(err).NotTo(HaveOccurred())
defer os.RemoveAll(tempDir)
// Launch the OSN without channels
ordererProcess := launchOrderer(gt, orderer, tempDir, configPath, "", cryptoPath, "none", "true", "info:orderer.common.server=debug")
defer func() { gt.Eventually(ordererProcess.Kill(), time.Minute).Should(gexec.Exit()) }()
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("Channel Participation API enabled, registrar initializing with file repo"))
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("Beginning to serve requests"))
joinBlockPath := writeJoinBlock(gt, configPath, configtxgen, tempDir)
gt.Eventually(ordererProcess.Kill(), time.Minute).Should(gexec.Exit())
// Restart, should pick up the join-block and start consensus on "mychannel"
ordererProcess = launchOrderer(gt, orderer, tempDir, configPath, "", cryptoPath, "none", "true", "info")
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("Channel Participation API enabled, registrar initializing with file repo"))
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("Starting Raft node channel=mychannel node=1"))
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("Beginning to serve requests"))
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("Raft leader changed: 0 -> 1 channel=mychannel node=1"))
// File was removed after on-boarding
_, err = os.Stat(joinBlockPath)
gt.Expect(err).To(HaveOccurred())
pathErr := err.(*os.PathError)
gt.Expect(pathErr.Err.Error()).To(Equal("no such file or directory"))
gt.Eventually(ordererProcess.Kill(), time.Minute).Should(gexec.Exit())
}
// emulate a join-block for an application channel written to the join-block filerepo location
func writeJoinBlock(gt *GomegaWithT, configPath string, configtxgen string, tempDir string) string {
genesisBlockPath := generateBootstrapBlock(gt, configPath, configtxgen, "mychannel", "SampleOrgChannel")
genesisBlockBytes, err := ioutil.ReadFile(genesisBlockPath)
gt.Expect(err).NotTo(HaveOccurred())
fileRepoDir := filepath.Join(tempDir, "ledger", "pendingops", "join")
err = os.MkdirAll(fileRepoDir, 0o744)
gt.Expect(err).NotTo(HaveOccurred())
joinBlockPath := filepath.Join(fileRepoDir, "mychannel.join")
err = ioutil.WriteFile(joinBlockPath, genesisBlockBytes, 0o644)
gt.Expect(err).NotTo(HaveOccurred())
return joinBlockPath
}
func testEtcdRaftOSNSuccess(gt *GomegaWithT, configPath, configtxgen, orderer, cryptoPath string) {
tempDir, err := ioutil.TempDir("", "etcdraft-test")
gt.Expect(err).NotTo(HaveOccurred())
defer os.RemoveAll(tempDir)
_ = writeJoinBlock(gt, configPath, configtxgen, tempDir)
// Launch the OSN
ordererProcess := launchOrderer(gt, orderer, tempDir, configPath, "", cryptoPath, "none", "true", "info")
defer func() { gt.Eventually(ordererProcess.Kill(), time.Minute).Should(gexec.Exit()) }()
// The following configuration parameters are not specified in the orderer.yaml, so let's ensure
// they are really configured autonomously via the localconfig code.
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("General.Cluster.DialTimeout = 5s"))
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("General.Cluster.RPCTimeout = 7s"))
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("General.Cluster.ReplicationBufferSize = 20971520"))
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("General.Cluster.ReplicationPullTimeout = 5s"))
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("General.Cluster.ReplicationRetryTimeout = 5s"))
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("General.Cluster.ReplicationBackgroundRefreshInterval = 5m0s"))
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("General.Cluster.ReplicationMaxRetries = 12"))
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("General.Cluster.SendBufferSize = 100"))
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("General.Cluster.CertExpirationWarningThreshold = 168h0m0s"))
// Consensus.EvictionSuspicion is not specified in orderer.yaml, so let's ensure
// it is really configured autonomously via the etcdraft chain itself.
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("EvictionSuspicion not set, defaulting to 10m"))
// Wait until the node starts up and elects itself as a single leader in a single node cluster.
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("Beginning to serve requests"))
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("becomeLeader"))
}
// the orderer refuses to launch with the 'file' bootstrap method
func testEtcdRaftOSNFailureInvalidBootstrapMethod(gt *GomegaWithT, configPath, orderer, configtxgen, cryptoPath string) {
tempDir, err := ioutil.TempDir("", "etcdraft-test")
gt.Expect(err).NotTo(HaveOccurred())
defer os.RemoveAll(tempDir)
// genesis block is ignored
genesisBlockPath := filepath.Join(tempDir, "does-not-exist", "genesis.block")
// Launch the OSN
ordererProcess := launchOrderer(gt, orderer, tempDir, configPath, genesisBlockPath, cryptoPath, "file", "true", "info")
defer func() { gt.Eventually(ordererProcess.Kill(), time.Minute).Should(gexec.Exit()) }()
expectedErr := "Bootstrap method: 'file' is forbidden, since system channel is no longer supported"
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say(expectedErr))
}
// the orderer ignores General.ChannelParticipation.Enabled=false
func testEtcdRaftOSNFailureChannelParticipationDisabled(gt *GomegaWithT, configPath, orderer, configtxgen, cryptoPath string) {
tempDir, err := ioutil.TempDir("", "etcdraft-test")
gt.Expect(err).NotTo(HaveOccurred())
defer os.RemoveAll(tempDir)
// genesis block is ignored
genesisBlockPath := filepath.Join(tempDir, "does-not-exist", "genesis.block")
// Launch the OSN
ordererProcess := launchOrderer(gt, orderer, tempDir, configPath, genesisBlockPath, cryptoPath, "none", "false", "info")
defer func() { gt.Eventually(ordererProcess.Kill(), time.Minute).Should(gexec.Exit()) }()
expectedErr := "General.ChannelParticipation.Enabled was set to false, setting to true"
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say(expectedErr))
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("Beginning to serve requests"))
}
func testEtcdRaftOSNNoTLSSingleListener(gt *GomegaWithT, configPath, orderer string, configtxgen, cryptoPath string) {
tempDir, err := ioutil.TempDir("", "etcdraft-test")
gt.Expect(err).NotTo(HaveOccurred())
defer os.RemoveAll(tempDir)
cmd := exec.Command(orderer)
cmd.Env = []string{
fmt.Sprintf("ORDERER_GENERAL_LISTENPORT=%d", nextPort()),
"ORDERER_GENERAL_BOOTSTRAPMETHOD=none",
fmt.Sprintf("ORDERER_FILELEDGER_LOCATION=%s", filepath.Join(tempDir, "ledger")),
fmt.Sprintf("FABRIC_CFG_PATH=%s", configPath),
}
ordererProcess, err := gexec.Start(cmd, nil, nil)
gt.Expect(err).NotTo(HaveOccurred())
defer func() { gt.Eventually(ordererProcess.Kill(), time.Minute).Should(gexec.Exit()) }()
expectedErr := "TLS is required for running ordering nodes of cluster type."
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say(expectedErr))
}
func testEtcdRaftOSNNoTLSDualListener(gt *GomegaWithT, configPath, orderer string, configtxgen, cryptoPath string) {
tempDir, err := ioutil.TempDir("", "etcdraft-test")
gt.Expect(err).NotTo(HaveOccurred())
defer os.RemoveAll(tempDir)
ordererTLSPath := filepath.Join(cryptoPath, "ordererOrganizations", "example.com", "orderers", "127.0.0.1.example.com", "tls")
_ = writeJoinBlock(gt, configPath, configtxgen, tempDir)
cmd := exec.Command(orderer)
cmd.Env = []string{
fmt.Sprintf("ORDERER_GENERAL_LISTENPORT=%d", nextPort()),
"ORDERER_GENERAL_BOOTSTRAPMETHOD=none",
"ORDERER_GENERAL_TLS_ENABLED=false",
"ORDERER_OPERATIONS_TLS_ENABLED=false",
fmt.Sprintf("ORDERER_FILELEDGER_LOCATION=%s", filepath.Join(tempDir, "ledger")),
fmt.Sprintf("ORDERER_GENERAL_CLUSTER_LISTENPORT=%d", nextPort()),
"ORDERER_GENERAL_CLUSTER_LISTENADDRESS=127.0.0.1",
fmt.Sprintf("ORDERER_GENERAL_CLUSTER_SERVERCERTIFICATE=%s", filepath.Join(ordererTLSPath, "server.crt")),
fmt.Sprintf("ORDERER_GENERAL_CLUSTER_SERVERPRIVATEKEY=%s", filepath.Join(ordererTLSPath, "server.key")),
fmt.Sprintf("ORDERER_GENERAL_CLUSTER_CLIENTCERTIFICATE=%s", filepath.Join(ordererTLSPath, "server.crt")),
fmt.Sprintf("ORDERER_GENERAL_CLUSTER_CLIENTPRIVATEKEY=%s", filepath.Join(ordererTLSPath, "server.key")),
fmt.Sprintf("ORDERER_GENERAL_CLUSTER_ROOTCAS=[%s]", filepath.Join(ordererTLSPath, "ca.crt")),
fmt.Sprintf("ORDERER_CONSENSUS_WALDIR=%s", filepath.Join(tempDir, "wal")),
fmt.Sprintf("ORDERER_CONSENSUS_SNAPDIR=%s", filepath.Join(tempDir, "snapshot")),
fmt.Sprintf("FABRIC_CFG_PATH=%s", configPath),
"ORDERER_OPERATIONS_LISTENADDRESS=127.0.0.1:0",
"ORDERER_CHANNELPARTICIPATION_ENABLED=true",
"FABRIC_LOGGING_SPEC=info",
}
ordererProcess, err := gexec.Start(cmd, nil, nil)
gt.Expect(err).NotTo(HaveOccurred())
defer func() { gt.Eventually(ordererProcess.Kill(), time.Minute).Should(gexec.Exit()) }()
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("Beginning to serve requests"))
gt.Eventually(ordererProcess.Err, time.Minute).Should(gbytes.Say("becomeLeader"))
}
func launchOrderer(gt *GomegaWithT, orderer, tempDir, configPath, genesisBlockPath, cryptoPath, bootstrapMethod, channelParticipationEnabled, logSpec string) *gexec.Session {
ordererTLSPath := filepath.Join(cryptoPath, "ordererOrganizations", "example.com", "orderers", "127.0.0.1.example.com", "tls")
// Launch the orderer process
cmd := exec.Command(orderer)
cmd.Env = []string{
fmt.Sprintf("ORDERER_GENERAL_LISTENPORT=%d", nextPort()),
"ORDERER_GENERAL_BOOTSTRAPMETHOD=" + bootstrapMethod,
"ORDERER_GENERAL_SYSTEMCHANNEL=system",
"ORDERER_GENERAL_TLS_CLIENTAUTHREQUIRED=true",
"ORDERER_GENERAL_TLS_ENABLED=true",
"ORDERER_OPERATIONS_TLS_ENABLED=false",
"ORDERER_FILELEDGER_LOCATION=" + filepath.Join(tempDir, "ledger"),
"ORDERER_GENERAL_BOOTSTRAPFILE=" + genesisBlockPath,
fmt.Sprintf("ORDERER_GENERAL_CLUSTER_LISTENPORT=%d", nextPort()),
"ORDERER_GENERAL_CLUSTER_LISTENADDRESS=127.0.0.1",
"ORDERER_GENERAL_CLUSTER_SERVERCERTIFICATE=" + filepath.Join(ordererTLSPath, "server.crt"),
"ORDERER_GENERAL_CLUSTER_SERVERPRIVATEKEY=" + filepath.Join(ordererTLSPath, "server.key"),
"ORDERER_GENERAL_CLUSTER_CLIENTCERTIFICATE=" + filepath.Join(ordererTLSPath, "server.crt"),
"ORDERER_GENERAL_CLUSTER_CLIENTPRIVATEKEY=" + filepath.Join(ordererTLSPath, "server.key"),
fmt.Sprintf("ORDERER_GENERAL_CLUSTER_ROOTCAS=[%s]", filepath.Join(ordererTLSPath, "ca.crt")),
fmt.Sprintf("ORDERER_GENERAL_TLS_ROOTCAS=[%s]", filepath.Join(ordererTLSPath, "ca.crt")),
"ORDERER_GENERAL_TLS_CERTIFICATE=" + filepath.Join(ordererTLSPath, "server.crt"),
"ORDERER_GENERAL_TLS_PRIVATEKEY=" + filepath.Join(ordererTLSPath, "server.key"),
"ORDERER_CONSENSUS_WALDIR=" + filepath.Join(tempDir, "wal"),
"ORDERER_CONSENSUS_SNAPDIR=" + filepath.Join(tempDir, "snapshot"),
"ORDERER_CHANNELPARTICIPATION_ENABLED=" + channelParticipationEnabled,
"FABRIC_CFG_PATH=" + configPath,
"FABRIC_LOGGING_SPEC=" + logSpec,
}
sess, err := gexec.Start(cmd, nil, nil)
gt.Expect(err).NotTo(HaveOccurred())
return sess
}