From dc33c15e4cfc69a00b1f896f980871a0bf0687bf Mon Sep 17 00:00:00 2001 From: Samu Tamminen Date: Fri, 19 Nov 2021 12:20:16 +0800 Subject: [PATCH 01/17] Refactor duplicate URL parsing code. Add new "cluster" entry to yaml and commanline arguments for alternative cluster start-up routine. --- pkg/option/option.go | 159 ++++++++++++++++++++++++++----------------- 1 file changed, 97 insertions(+), 62 deletions(-) diff --git a/pkg/option/option.go b/pkg/option/option.go index f81104cf26..324f0a4241 100644 --- a/pkg/option/option.go +++ b/pkg/option/option.go @@ -35,7 +35,16 @@ import ( "github.com/megaease/easegress/pkg/version" ) -// Options is the startup options. +// ClusterOptions is the start-up options. +type ClusterOptions struct { + ListenPeerURLs []string `yaml:"listen-peer-urls"` + ListenClientURLs []string `yaml:"listen-client-urls"` + AdvertiseClientURLs []string `yaml:"advertise-client-urls"` + InitialAdvertisePeerURLs []string `yaml:"initial-advertise-peer-urls"` + InitialCluster map[string]string `yaml:"initial-cluster"` +} + +// Options is the start-up options. type Options struct { flags *pflag.FlagSet viper *viper.Viper @@ -54,6 +63,12 @@ type Options struct { // meta Name string `yaml:"name" env:"EG_NAME"` Labels map[string]string `yaml:"labels" env:"EG_LABELS"` + APIAddr string `yaml:"api-addr"` + Debug bool `yaml:"debug"` + DisableAccessLog bool `yaml:"disable-access-log"` + InitialObjectConfigFiles []string `yaml:"initial-object-config-files"` + + // cluster options ClusterName string `yaml:"cluster-name"` ClusterRole string `yaml:"cluster-role"` ClusterRequestTimeout string `yaml:"cluster-request-timeout"` @@ -62,10 +77,7 @@ type Options struct { ClusterAdvertiseClientURLs []string `yaml:"cluster-advertise-client-urls"` ClusterInitialAdvertisePeerURLs []string `yaml:"cluster-initial-advertise-peer-urls"` ClusterJoinURLs []string `yaml:"cluster-join-urls"` - APIAddr string `yaml:"api-addr"` - Debug bool `yaml:"debug"` - DisableAccessLog bool `yaml:"disable-access-log"` - InitialObjectConfigFiles []string `yaml:"initial-object-config-files"` + Cluster ClusterOptions `yaml:"cluster"` // Path. HomeDir string `yaml:"home-dir"` @@ -86,6 +98,29 @@ type Options struct { AbsMemberDir string `yaml:"-"` } +// addClusterVars introduces cluster arguments. +func addClusterVars(opt *Options) { + opt.flags.StringVar(&opt.ClusterName, "cluster-name", "eg-cluster-default-name", "Human-readable name for the new cluster, ignored while joining an existed cluster.") + opt.flags.StringVar(&opt.ClusterRole, "cluster-role", "writer", "Cluster role for this member (reader, writer).") + opt.flags.StringVar(&opt.ClusterRequestTimeout, "cluster-request-timeout", "10s", "Timeout to handle request in the cluster.") + + // Cluster connection configuration style 1 + opt.flags.StringSliceVar(&opt.ClusterListenClientURLs, "cluster-listen-client-urls", []string{"http://localhost:2379"}, "List of URLs to listen on for cluster client traffic.") + opt.flags.StringSliceVar(&opt.ClusterListenPeerURLs, "cluster-listen-peer-urls", []string{"http://localhost:2380"}, "List of URLs to listen on for cluster peer traffic.") + opt.flags.StringSliceVar(&opt.ClusterAdvertiseClientURLs, "cluster-advertise-client-urls", []string{"http://localhost:2379"}, "List of this member’s client URLs to advertise to the rest of the cluster.") + opt.flags.StringSliceVar(&opt.ClusterInitialAdvertisePeerURLs, "cluster-initial-advertise-peer-urls", []string{"http://localhost:2380"}, "List of this member’s peer URLs to advertise to the rest of the cluster.") + opt.flags.StringSliceVar(&opt.ClusterJoinURLs, "cluster-join-urls", nil, + "List of URLs to join, when the first url is the same with any one of cluster-initial-advertise-peer-urls, it means to join itself, and this config will be treated empty. When used, leave initial-cluster empty.") + + // Cluster connection configuration style 2 + opt.flags.StringSliceVar(&opt.Cluster.ListenClientURLs, "listen-client-urls", []string{"http://localhost:2379"}, "List of URLs to listen on for cluster client traffic.") + opt.flags.StringSliceVar(&opt.Cluster.ListenPeerURLs, "listen-peer-urls", []string{"http://localhost:2380"}, "List of URLs to listen on for cluster peer traffic.") + opt.flags.StringSliceVar(&opt.Cluster.AdvertiseClientURLs, "advertise-client-urls", []string{"http://localhost:2379"}, "List of this member’s client URLs to advertise to the rest of the cluster.") + opt.flags.StringSliceVar(&opt.Cluster.InitialAdvertisePeerURLs, "initial-advertise-peer-urls", []string{"http://localhost:2380"}, "List of this member’s peer URLs to advertise to the rest of the cluster.") + opt.flags.StringToStringVarP(&opt.Cluster.InitialCluster, "initial-cluster", "", nil, + "List of (member name, URL) pairs that will form the cluster. E.g. writer-1=http://localhost:2380. When used, leave cluster-join-urls empty.") +} + // New creates a default Options. func New() *Options { opt := &Options{ @@ -101,14 +136,7 @@ func New() *Options { opt.flags.BoolVar(&opt.SignalUpgrade, "signal-upgrade", false, "Send an upgrade signal to the server based on the local pid file, then exit. The original server will start a graceful upgrade after signal received.") opt.flags.StringVar(&opt.Name, "name", "eg-default-name", "Human-readable name for this member.") opt.flags.StringToStringVar(&opt.Labels, "labels", nil, "The labels for the instance of Easegress.") - opt.flags.StringVar(&opt.ClusterName, "cluster-name", "eg-cluster-default-name", "Human-readable name for the new cluster, ignored while joining an existed cluster.") - opt.flags.StringVar(&opt.ClusterRole, "cluster-role", "writer", "Cluster role for this member (reader, writer).") - opt.flags.StringVar(&opt.ClusterRequestTimeout, "cluster-request-timeout", "10s", "Timeout to handle request in the cluster.") - opt.flags.StringSliceVar(&opt.ClusterListenClientURLs, "cluster-listen-client-urls", []string{"http://localhost:2379"}, "List of URLs to listen on for cluster client traffic.") - opt.flags.StringSliceVar(&opt.ClusterListenPeerURLs, "cluster-listen-peer-urls", []string{"http://localhost:2380"}, "List of URLs to listen on for cluster peer traffic.") - opt.flags.StringSliceVar(&opt.ClusterAdvertiseClientURLs, "cluster-advertise-client-urls", []string{"http://localhost:2379"}, "List of this member’s client URLs to advertise to the rest of the cluster.") - opt.flags.StringSliceVar(&opt.ClusterInitialAdvertisePeerURLs, "cluster-initial-advertise-peer-urls", []string{"http://localhost:2380"}, "List of this member’s peer URLs to advertise to the rest of the cluster.") - opt.flags.StringSliceVar(&opt.ClusterJoinURLs, "cluster-join-urls", nil, "List of URLs to join, when the first url is the same with any one of cluster-initial-advertise-peer-urls, it means to join itself, and this config will be treated empty.") + addClusterVars(opt) opt.flags.StringVar(&opt.APIAddr, "api-addr", "localhost:2381", "Address([host]:port) to listen on for administration traffic.") opt.flags.BoolVar(&opt.Debug, "debug", false, "Flag to set lowest log level from INFO downgrade DEBUG.") opt.flags.StringSliceVar(&opt.InitialObjectConfigFiles, "initial-object-config-files", nil, "List of configuration files for initial objects, these objects will be created at startup if not already exist.") @@ -132,6 +160,11 @@ func (opt *Options) YAML() string { return opt.yamlStr } +// UseInitialCluster returns true if the cluster.initial-cluster is defined. If it is, the ClusterJoinUrls is ignored. +func (opt *Options) UseInitialCluster() bool { + return len(opt.Cluster.InitialCluster) > 0 +} + // Parse parses all arguments, returns normal message without error if --help/--version set. func (opt *Options) Parse() (string, error) { err := opt.flags.Parse(os.Args[1:]) @@ -205,7 +238,7 @@ func (opt *Options) Parse() (string, error) { // adjust adjusts the options to handle conflict // between user's config and internal component. func (opt *Options) adjust() { - if opt.ClusterRole != "writer" { + if opt.ClusterRole != "writer" || opt.UseInitialCluster() { return } if len(opt.ClusterJoinURLs) == 0 { @@ -225,75 +258,77 @@ func (opt *Options) adjust() { } } +// ParseURLs parses list of strings to url.URL objects. +func ParseURLs(urlStrings []string) ([]url.URL, error) { + urls := make([]url.URL, len(urlStrings)) + for i, urlString := range urlStrings { + parsedUrl, err := url.Parse(urlString) + if err != nil { + fmt.Println(urlString) + return nil, fmt.Errorf(" %s: %v", urlString, err) + } + urls[i] = *parsedUrl + } + return urls, nil +} + func (opt *Options) validate() error { if opt.ClusterName == "" { return fmt.Errorf("empty cluster-name") } else if err := common.ValidateName(opt.ClusterName); err != nil { return err } - + if len(opt.ClusterJoinURLs) != 0 { + if _, err := ParseURLs(opt.ClusterJoinURLs); err != nil { + return fmt.Errorf("invalid cluster-join-urls: %v", err) + } + } switch opt.ClusterRole { case "reader": if opt.ForceNewCluster { return fmt.Errorf("reader got force-new-cluster") } - if len(opt.ClusterJoinURLs) == 0 { return fmt.Errorf("reader got empty cluster-join-urls") } - - for _, urlText := range opt.ClusterJoinURLs { - _, err := url.Parse(urlText) - if err != nil { - return fmt.Errorf("invalid cluster-join-urls: %v", err) - } - } - case "writer": - if len(opt.ClusterListenClientURLs) == 0 { - return fmt.Errorf("empty cluster-listen-client-urls") - } - if len(opt.ClusterListenPeerURLs) == 0 { - return fmt.Errorf("empty cluster-listen-peer-urls") - } - if len(opt.ClusterAdvertiseClientURLs) == 0 { - return fmt.Errorf("empty cluster-advertise-client-urls") - } - if len(opt.ClusterInitialAdvertisePeerURLs) == 0 { - return fmt.Errorf("empty cluster-initial-advertise-peer-urls") + argumentsToValidate := map[string][]string{ + "cluster-listen-client-urls": opt.ClusterListenClientURLs, + "cluster-listen-peer-urls": opt.ClusterListenPeerURLs, + "cluster-advertise-client-urls": opt.ClusterAdvertiseClientURLs, + "cluster-initial-advertise-peer-urls": opt.ClusterInitialAdvertisePeerURLs, } - for _, clientURL := range opt.ClusterListenClientURLs { - _, err := url.Parse(clientURL) - if err != nil { - return fmt.Errorf("invalid cluster-listen-client-urls: %s: %v", clientURL, err) + if opt.UseInitialCluster() { + argumentsToValidate = map[string][]string{ + "listen-client-urls": opt.Cluster.ListenClientURLs, + "listen-peer-urls": opt.Cluster.ListenPeerURLs, + "advertise-client-urls": opt.Cluster.AdvertiseClientURLs, + "initial-advertise-peer-urls": opt.Cluster.InitialAdvertisePeerURLs, } - } - for _, peerURL := range opt.ClusterListenPeerURLs { - _, err := url.Parse(peerURL) - if err != nil { - return fmt.Errorf("invalid cluster-listen-peer-urls: %s: %v", peerURL, err) + initialClusterUrls := make([]string, 0, len(opt.Cluster.InitialCluster)) + for _, value := range opt.Cluster.InitialCluster { + initialClusterUrls = append(initialClusterUrls, value) } - } - for _, clientURL := range opt.ClusterAdvertiseClientURLs { - _, err := url.Parse(clientURL) - if err != nil { - return fmt.Errorf("invalid cluster-advertise-client-urls: %s: %v", clientURL, err) + if _, err := ParseURLs(initialClusterUrls); err != nil { + return fmt.Errorf("invalid initial-cluster: %v", err) } - } - for _, peerURL := range opt.ClusterInitialAdvertisePeerURLs { - _, err := url.Parse(peerURL) - if err != nil { - return fmt.Errorf("invalid cluster-initial-advertise-peer-urls: %s: %v", peerURL, err) + if len(opt.ClusterJoinURLs) > 0 { + err := strings.Join(strings.Fields(` + %s and %s are both defined. + Please provide only one of them. %s is the recommended way.`), " ") + if len(opt.ConfigFile) == 0 { + return fmt.Errorf(err, "--initial-cluster", "--cluster-join-urls", "--initial-cluster") + } + return fmt.Errorf(err, "cluster.initial-cluster", "cluster-join-urls", "cluster.initial-cluster") } } - - if len(opt.ClusterJoinURLs) != 0 { - for _, joinURL := range opt.ClusterJoinURLs { - _, err := url.Parse(joinURL) - if err != nil { - return fmt.Errorf("invalid cluster-join-urls: %s: %v", joinURL, err) - } + for arg, urls := range argumentsToValidate { + if len(urls) == 0 { + return fmt.Errorf("empty %s", arg) + } + if _, err := ParseURLs(urls); err != nil { + return fmt.Errorf("invalid %s: %v", arg, err) } } default: @@ -324,7 +359,7 @@ func (opt *Options) validate() error { if opt.LogDir == "" { return fmt.Errorf("empty log-dir") } - if opt.MemberDir == "" { + if !opt.UseInitialCluster() && opt.MemberDir == "" { return fmt.Errorf("empty member-dir") } From d43ba7a65a3b96bf57ffed5d8735c714f8dbe693 Mon Sep 17 00:00:00 2001 From: Samu Tamminen Date: Fri, 19 Nov 2021 14:09:06 +0800 Subject: [PATCH 02/17] configuration for static sized cluster --- pkg/cluster/cluster.go | 7 ++- pkg/cluster/config.go | 114 ++++++++++++++++++++++++++----------- pkg/cluster/config_test.go | 74 ++++++++++++++++++++++++ 3 files changed, 160 insertions(+), 35 deletions(-) create mode 100644 pkg/cluster/config_test.go diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 046cee6080..08b8a32e61 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -632,8 +632,13 @@ func (c *cluster) startServer() (done, timeout chan struct{}, err error) { close(done) return done, timeout, nil } + createConfig := CreateEtcdConfig + if c.opt.UseInitialCluster() { + createConfig = CreateStaticClusterEtcdConfig + } + // TODO disable members file - etcdConfig, err := c.prepareEtcdConfig() + etcdConfig, err := createConfig(c.opt, c.members) if err != nil { return nil, nil, err } diff --git a/pkg/cluster/config.go b/pkg/cluster/config.go index bf1d52be8a..e7486f2d7e 100644 --- a/pkg/cluster/config.go +++ b/pkg/cluster/config.go @@ -26,6 +26,7 @@ import ( "github.com/megaease/easegress/pkg/common" "github.com/megaease/easegress/pkg/logger" + "github.com/megaease/easegress/pkg/option" ) const ( @@ -46,43 +47,88 @@ var ( autoCompactionMode = embed.CompactorModeRevision ) -func (c *cluster) prepareEtcdConfig() (*embed.Config, error) { +// CreateStaticClusterEtcdConfig creates an embedded etcd config for static sized cluster, +// listing all cluster members for etcd's initial-cluster argument. +func CreateStaticClusterEtcdConfig(opt *option.Options, members *members) (*embed.Config, error) { ec := embed.NewConfig() - opt := c.opt var ( clientURLs []url.URL peerURLs []url.URL clientAdURLs []url.URL peerAdURLs []url.URL + ) - for _, u := range opt.ClusterListenClientURLs { - clientURL, err := url.Parse(u) - if err != nil { - return nil, err - } - clientURLs = append(clientURLs, *clientURL) + clientURLs,err := option.ParseURLs(opt.Cluster.ListenClientURLs) + if err != nil { + return nil, err } - for _, u := range opt.ClusterListenPeerURLs { - peerURL, err := url.Parse(u) - if err != nil { - return nil, err - } - peerURLs = append(peerURLs, *peerURL) + peerURLs,err = option.ParseURLs(opt.Cluster.ListenPeerURLs) + if err != nil { + return nil, err } - for _, u := range opt.ClusterAdvertiseClientURLs { - clientAdURL, err := url.Parse(u) - if err != nil { - return nil, err - } - clientAdURLs = append(clientAdURLs, *clientAdURL) + clientAdURLs,err = option.ParseURLs(opt.Cluster.AdvertiseClientURLs) + if err != nil { + return nil, err } - for _, u := range opt.ClusterInitialAdvertisePeerURLs { - peerAdURL, err := url.Parse(u) - if err != nil { - return nil, err - } - peerAdURLs = append(peerAdURLs, *peerAdURL) + peerAdURLs,err = option.ParseURLs(opt.Cluster.InitialAdvertisePeerURLs) + if err != nil { + return nil, err + } + + ec.Name = opt.Name + + ec.Dir = opt.AbsDataDir + ec.WalDir = opt.AbsWALDir + ec.InitialClusterToken = opt.ClusterName + ec.EnableV2 = false + ec.LCUrls = clientURLs + ec.ACUrls = clientAdURLs + ec.LPUrls = peerURLs + ec.APUrls = peerAdURLs + ec.AutoCompactionMode = autoCompactionMode + ec.AutoCompactionRetention = autoCompactionRetention + ec.QuotaBackendBytes = quotaBackendBytes + ec.MaxTxnOps = maxTxnOps + ec.MaxRequestBytes = maxRequestBytes + ec.Logger = "zap" + ec.LogOutputs = []string{common.NormalizeZapLogPath(filepath.Join(opt.AbsLogDir, logFilename))} + + ec.ClusterState = embed.ClusterStateFlagNew + ec.InitialCluster = members.initCluster() + + logger.Infof("etcd config: init-cluster:%s cluster-state:%s force-new-cluster:%v", + ec.InitialCluster, ec.ClusterState, ec.ForceNewCluster) + + return ec, nil +} + + +// CreateEtcdConfig creates an embedded etcd config that starts the cluster by adding member by member. +func CreateEtcdConfig(opt *option.Options, members *members) (*embed.Config, error) { + ec := embed.NewConfig() + + var ( + clientURLs []url.URL + peerURLs []url.URL + clientAdURLs []url.URL + peerAdURLs []url.URL + ) + clientURLs,err := option.ParseURLs(opt.ClusterListenClientURLs) + if err != nil { + return nil, err + } + peerURLs,err = option.ParseURLs(opt.ClusterListenPeerURLs) + if err != nil { + return nil, err + } + clientAdURLs,err = option.ParseURLs(opt.ClusterAdvertiseClientURLs) + if err != nil { + return nil, err + } + peerAdURLs,err = option.ParseURLs(opt.ClusterInitialAdvertisePeerURLs) + if err != nil { + return nil, err } ec.Name = opt.Name @@ -104,22 +150,22 @@ func (c *cluster) prepareEtcdConfig() (*embed.Config, error) { ec.LogOutputs = []string{common.NormalizeZapLogPath(filepath.Join(opt.AbsLogDir, logFilename))} ec.ClusterState = embed.ClusterStateFlagExisting - if c.opt.ForceNewCluster { + if opt.ForceNewCluster { ec.ClusterState = embed.ClusterStateFlagNew ec.ForceNewCluster = true - self := c.members.self() + self := members.self() ec.InitialCluster = fmt.Sprintf("%s=%s", self.Name, self.PeerURL) } else { - if len(c.opt.ClusterJoinURLs) == 0 { - if c.members.clusterMembersLen() == 1 && - common.IsDirEmpty(c.opt.AbsDataDir) { + if len(opt.ClusterJoinURLs) == 0 { + if members.clusterMembersLen() == 1 && + common.IsDirEmpty(opt.AbsDataDir) { ec.ClusterState = embed.ClusterStateFlagNew } - } else if c.members.clusterMembersLen() == 1 { + } else if members.clusterMembersLen() == 1 { return nil, fmt.Errorf("join mode with only one cluster member: %v", - *c.members.ClusterMembers) + *members.ClusterMembers) } - ec.InitialCluster = c.members.initCluster() + ec.InitialCluster = members.initCluster() } logger.Infof("etcd config: init-cluster:%s cluster-state:%s force-new-cluster:%v", diff --git a/pkg/cluster/config_test.go b/pkg/cluster/config_test.go new file mode 100644 index 0000000000..3ea3d40320 --- /dev/null +++ b/pkg/cluster/config_test.go @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2017, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cluster + +import ( + "fmt" + "testing" + "strings" + + "github.com/megaease/easegress/pkg/option" +) + +func TestCreateEtcdConfigFailures(t *testing.T) { + testData := make([]*option.Options, 0) + testData = append(testData, mockTestOpt()) + testData[len(testData)-1].ClusterListenClientURLs = []string{"::::::"} + testData = append(testData, mockTestOpt()) + testData[len(testData)-1].ClusterListenPeerURLs = []string{"::::::"} + testData = append(testData, mockTestOpt()) + testData[len(testData)-1].ClusterAdvertiseClientURLs = []string{"::::::"} + testData = append(testData, mockTestOpt()) + testData[len(testData)-1].ClusterInitialAdvertisePeerURLs = []string{"::::::"} + + for i, opt := range testData { + t.Run(fmt.Sprintf("CreateEtcdConfig: options invalid url i=%d",i), func(t *testing.T) { + membersInstance, _ := newMembers(opt) + _, err := CreateEtcdConfig(opt, membersInstance) + if err == nil { + t.Error("There should be an error") + } + if !strings.Contains(err.Error(), "missing protocol scheme") { + t.Error("Error should contain missing protocol scheme") + } + }) + } + + testData = make([]*option.Options, 0) + testData = append(testData, mockTestOpt()) + testData[len(testData)-1].Cluster.ListenClientURLs = []string{"::::::"} + testData = append(testData, mockTestOpt()) + testData[len(testData)-1].Cluster.ListenPeerURLs = []string{"::::::"} + testData = append(testData, mockTestOpt()) + testData[len(testData)-1].Cluster.AdvertiseClientURLs = []string{"::::::"} + testData = append(testData, mockTestOpt()) + testData[len(testData)-1].Cluster.InitialAdvertisePeerURLs = []string{"::::::"} + + for i, opt := range testData { + t.Run(fmt.Sprintf("CreateStaticClusterEtcdConfig: options invalid url i=%d",i), func(t *testing.T) { + membersInstance, _ := newMembers(opt) + _, err := CreateStaticClusterEtcdConfig(opt, membersInstance) + if err == nil { + t.Error("There should be an error") + } + if !strings.Contains(err.Error(), "missing protocol scheme") { + t.Error("Error should contain missing protocol scheme") + } + }) + } +} From 94d226bb250d1f8729f6675fa412f9b775d72f89 Mon Sep 17 00:00:00 2001 From: Samu Tamminen Date: Mon, 22 Nov 2021 09:02:29 +0800 Subject: [PATCH 03/17] examples for new cluster configuration --- example/config/cluster/cluster-1.yaml | 23 +++++++++++++++++++ example/config/cluster/cluster-2.yaml | 23 +++++++++++++++++++ example/config/cluster/cluster-3.yaml | 23 +++++++++++++++++++ example/config/cluster/legacy-cluster-1.yaml | 23 +++++++++++++++++++ example/config/cluster/legacy-cluster-2.yaml | 23 +++++++++++++++++++ example/config/cluster/legacy-cluster-3.yaml | 23 +++++++++++++++++++ .../config/cluster/no-cluster-example.yaml | 10 ++++++++ 7 files changed, 148 insertions(+) create mode 100644 example/config/cluster/cluster-1.yaml create mode 100644 example/config/cluster/cluster-2.yaml create mode 100644 example/config/cluster/cluster-3.yaml create mode 100644 example/config/cluster/legacy-cluster-1.yaml create mode 100644 example/config/cluster/legacy-cluster-2.yaml create mode 100644 example/config/cluster/legacy-cluster-3.yaml create mode 100644 example/config/cluster/no-cluster-example.yaml diff --git a/example/config/cluster/cluster-1.yaml b/example/config/cluster/cluster-1.yaml new file mode 100644 index 0000000000..bade277079 --- /dev/null +++ b/example/config/cluster/cluster-1.yaml @@ -0,0 +1,23 @@ +name: writer-1 +cluster-name: cluster-test +cluster-role: writer +api-addr: localhost:2831 +data-dir: ./data +wal-dir: "" +cpu-profile-file: +memory-profile-file: +log-dir: ./log +debug: false +cluster: + listen-peer-urls: + - http://127.0.0.1:2380 + listen-client-urls: + - http://127.0.0.1:2379 + advertise-client-urls: + - http://127.0.0.1:2379 + initial-advertise-peer-urls: + - http://127.0.0.1:2380 + initial-cluster: + - writer-1: http://127.0.0.1:2380 + - writer-2: http://127.0.0.1:2378 + - writer-3: http://127.0.0.1:2376 diff --git a/example/config/cluster/cluster-2.yaml b/example/config/cluster/cluster-2.yaml new file mode 100644 index 0000000000..94ea238d95 --- /dev/null +++ b/example/config/cluster/cluster-2.yaml @@ -0,0 +1,23 @@ +name: writer-2 +cluster-name: cluster-test +cluster-role: writer +api-addr: localhost:2831 +data-dir: ./data +wal-dir: "" +cpu-profile-file: +memory-profile-file: +log-dir: ./log +debug: false +cluster: + listen-peer-urls: + - http://127.0.0.1:2378 + listen-client-urls: + - http://127.0.0.1:2377 + advertise-client-urls: + - http://127.0.0.1:2377 + initial-advertise-peer-urls: + - http://127.0.0.1:2378 + initial-cluster: + - writer-1: http://127.0.0.1:2380 + - writer-2: http://127.0.0.1:2378 + - writer-3: http://127.0.0.1:2376 diff --git a/example/config/cluster/cluster-3.yaml b/example/config/cluster/cluster-3.yaml new file mode 100644 index 0000000000..74156053b9 --- /dev/null +++ b/example/config/cluster/cluster-3.yaml @@ -0,0 +1,23 @@ +name: writer-3 +cluster-name: cluster-test +cluster-role: writer +api-addr: localhost:2831 +data-dir: ./data +wal-dir: "" +cpu-profile-file: +memory-profile-file: +log-dir: ./log +debug: false +cluster: + listen-peer-urls: + - http://127.0.0.1:2376 + listen-client-urls: + - http://127.0.0.1:2375 + advertise-client-urls: + - http://127.0.0.1:2375 + initial-advertise-peer-urls: + - http://127.0.0.1:2376 + initial-cluster: + - writer-1: http://127.0.0.1:2380 + - writer-2: http://127.0.0.1:2378 + - writer-3: http://127.0.0.1:2376 diff --git a/example/config/cluster/legacy-cluster-1.yaml b/example/config/cluster/legacy-cluster-1.yaml new file mode 100644 index 0000000000..2e4035bf17 --- /dev/null +++ b/example/config/cluster/legacy-cluster-1.yaml @@ -0,0 +1,23 @@ +name: writer-1 +cluster-name: cluster-test +cluster-role: writer +api-addr: localhost:2831 +data-dir: ./data +wal-dir: "" +cpu-profile-file: +memory-profile-file: +log-dir: ./log +debug: false +cluster-listen-client-urls: + - http://127.0.0.1:2379 +cluster-listen-peer-urls: +- http://127.0.0.1:2380 +cluster-advertise-client-urls: + - http://127.0.0.1:2379 +cluster-initial-advertise-peer-urls: + - http://127.0.0.1:2380 +cluster-join-urls: + - http://127.0.0.1:2380 + - http://127.0.0.1:2378 + - http://127.0.0.1:2376 +member-dir: ./member diff --git a/example/config/cluster/legacy-cluster-2.yaml b/example/config/cluster/legacy-cluster-2.yaml new file mode 100644 index 0000000000..ea70242349 --- /dev/null +++ b/example/config/cluster/legacy-cluster-2.yaml @@ -0,0 +1,23 @@ +name: writer-2 +cluster-name: cluster-test +cluster-role: writer +api-addr: localhost:2831 +data-dir: ./data +wal-dir: "" +cpu-profile-file: +memory-profile-file: +log-dir: ./log +debug: false +cluster-listen-client-urls: + - http://127.0.0.1:2377 +cluster-listen-peer-urls: +- http://127.0.0.1:2378 +cluster-advertise-client-urls: + - http://127.0.0.1:2377 +cluster-initial-advertise-peer-urls: + - http://127.0.0.1:2378 +cluster-join-urls: + - http://127.0.0.1:2380 + - http://127.0.0.1:2378 + - http://127.0.0.1:2376 +member-dir: ./member diff --git a/example/config/cluster/legacy-cluster-3.yaml b/example/config/cluster/legacy-cluster-3.yaml new file mode 100644 index 0000000000..79103b9d07 --- /dev/null +++ b/example/config/cluster/legacy-cluster-3.yaml @@ -0,0 +1,23 @@ +name: writer-3 +cluster-name: cluster-test +cluster-role: writer +api-addr: localhost:2831 +data-dir: ./data +wal-dir: "" +cpu-profile-file: +memory-profile-file: +log-dir: ./log +debug: false +cluster-listen-client-urls: + - http://127.0.0.1:2375 +cluster-listen-peer-urls: +- http://127.0.0.1:2376 +cluster-advertise-client-urls: + - http://127.0.0.1:2375 +cluster-initial-advertise-peer-urls: + - http://127.0.0.1:2376 +cluster-join-urls: + - http://127.0.0.1:2380 + - http://127.0.0.1:2378 + - http://127.0.0.1:2376 +member-dir: ./member diff --git a/example/config/cluster/no-cluster-example.yaml b/example/config/cluster/no-cluster-example.yaml new file mode 100644 index 0000000000..9ea04d1c3e --- /dev/null +++ b/example/config/cluster/no-cluster-example.yaml @@ -0,0 +1,10 @@ +name: writer-n +cluster-name: cluster-test +cluster-role: writer +api-addr: 0.0.0.0:2831 +data-dir: ./data +wal-dir: "" +cpu-profile-file: +memory-profile-file: +log-dir: ./log +debug: false From ae1748fe6af5690e123fbc7d843a45c52f3bf555 Mon Sep 17 00:00:00 2001 From: Samu Tamminen Date: Mon, 22 Nov 2021 09:06:08 +0800 Subject: [PATCH 04/17] Separate logic of members initialization. Add unittest for new configuration. --- pkg/cluster/cluster.go | 3 +- pkg/cluster/cluster_test.go | 53 ++++++++++++++++++++++++--- pkg/cluster/member.go | 41 +++++++++++++++++---- pkg/cluster/member_test.go | 73 +++++++++++++++++++++++++++++++++++++ 4 files changed, 156 insertions(+), 14 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 08b8a32e61..34f86e2e58 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -241,7 +241,7 @@ func (c *cluster) getReady() error { return nil } - if !c.opt.ForceNewCluster && c.members.knownMembersLen() > 1 { + if !c.opt.ForceNewCluster && c.members.knownMembersLen() > 1 && !c.opt.UseInitialCluster() { client, _ := c.getClient() if client != nil { err := c.addSelfToCluster() @@ -636,7 +636,6 @@ func (c *cluster) startServer() (done, timeout chan struct{}, err error) { if c.opt.UseInitialCluster() { createConfig = CreateStaticClusterEtcdConfig } - // TODO disable members file etcdConfig, err := createConfig(c.opt, c.members) if err != nil { diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 3f95b1c19e..3dd1d666ee 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -85,6 +85,43 @@ func mockClusters(count int) []*cluster { return clusters } +func mockStaticCluster(count int) []*cluster { + opts, _, _ := mockStaticClusterMembers(count) + + clusterNodes := make([]*cluster, count) + clusterNodesLock := sync.Mutex{} + clusterCreationWg := &sync.WaitGroup{} + clusterCreationWg.Add(count) + startNode := func(i int) error { + node, err := New(opts[i]) + if err != nil { + panic(fmt.Errorf("new cluster failed: %v", err)) + } + clusterNodesLock.Lock() + clusterNodes[i] = node.(*cluster) + clusterNodesLock.Unlock() + clusterCreationWg.Done() + return nil + } + + for i := 0; i < count; i++ { + go startNode(i) + } + clusterCreationWg.Wait() + + for { + _, err := clusterNodes[0].getClient() + time.Sleep(HeartbeatInterval) + if err != nil { + fmt.Println(err) + continue + } else { + break + } + } + return clusterNodes +} + func closeClusters(clusters []*cluster) { wg := &sync.WaitGroup{} wg.Add(len(clusters)) @@ -94,12 +131,18 @@ func closeClusters(clusters []*cluster) { } } -func TestCluster(t *testing.T) { - clusters := mockClusters(3) - defer closeClusters(clusters) - // for testing longRequestContext() - clusters[0].longRequestContext() +func TestCluster(t *testing.T) { + t.Run("start cluster dynamically", func(t *testing.T) { + clusters := mockClusters(3) + defer closeClusters(clusters) + // for testing longRequestContext() + clusters[0].longRequestContext() + }) + t.Run("start static sized cluster", func(t *testing.T) { + clusterNodes := mockStaticCluster(3) + defer closeClusters(clusterNodes) + }) } func TestLease(t *testing.T) { diff --git a/pkg/cluster/member.go b/pkg/cluster/member.go index ad58ed87ec..0b897a6d5c 100644 --- a/pkg/cluster/member.go +++ b/pkg/cluster/member.go @@ -72,9 +72,24 @@ func newMembers(opt *option.Options) (*members, error) { ClusterMembers: newMemberSlices(), KnownMembers: newMemberSlices(), } + if opt.UseInitialCluster() { + m.initializeStaticClusterMembers(opt) + } else { + m.initializeMembers(opt) + err := m.load() + if err != nil { + return nil, err + } + } + + return m, nil +} +// initializeMembers adds first member to ClusterMembers and all members to KnownMembers. +func (m *members) initializeMembers(opt *option.Options) { initMS := make(membersSlice, 0) if opt.ClusterRole == "writer" && len(opt.ClusterInitialAdvertisePeerURLs) != 0 { + // Cluster is started member by member --> start with cluster of size 1 initMS = append(initMS, &member{ Name: opt.Name, PeerURL: opt.ClusterInitialAdvertisePeerURLs[0], @@ -82,6 +97,7 @@ func newMembers(opt *option.Options) (*members, error) { } m.ClusterMembers.update(initMS) + // Add all members to list of known members if len(opt.ClusterJoinURLs) != 0 { for _, peerURL := range opt.ClusterJoinURLs { initMS = append(initMS, &member{ @@ -90,13 +106,19 @@ func newMembers(opt *option.Options) (*members, error) { } } m.KnownMembers.update(initMS) +} - err := m.load() - if err != nil { - return nil, err - } - - return m, nil +// initializeStaticClusterMembers adds all members to ClusterMembers and KnownMembers. +func (m *members) initializeStaticClusterMembers(opt *option.Options) { + initMS := make(membersSlice, 0) + for name, peerURL := range opt.Cluster.InitialCluster { + initMS = append(initMS, &member{ + Name: name, + PeerURL: peerURL, + }) + } + m.ClusterMembers.update(initMS) + m.KnownMembers.update(initMS) } func (m *members) fileExist() bool { @@ -223,7 +245,12 @@ func (m *members) updateClusterMembers(pbMembers []*pb.Member) { // NOTE: KnownMembers store members as many as possible m.KnownMembers.update(*m.ClusterMembers) - m.store() + if !m.opt.UseInitialCluster() { + // When cluster is initialized member by member, persist KnownMembers and ClusterMembers + // to disk for failure recovery. If a member fails for any reason before it has been added + // to cluster, the persisted file can be used to continue initialization. + m.store() + } } func (m *members) knownMembersLen() int { diff --git a/pkg/cluster/member_test.go b/pkg/cluster/member_test.go index 177d7eab21..5feaa4b332 100644 --- a/pkg/cluster/member_test.go +++ b/pkg/cluster/member_test.go @@ -133,6 +133,79 @@ func mockMembers(count int) ([]*option.Options, membersSlice, []*pb.Member) { return opts, members, pbMembers } +func mockStaticClusterMembers(count int) ([]*option.Options, membersSlice, []*pb.Member) { + opts := make([]*option.Options, count) + members := make(membersSlice, count) + pbMembers := make([]*pb.Member, count) + + portCount := (count * 2) + 1 // two for each member and one for egctl API. + ports, err := freeport.GetFreePorts(portCount) + if err != nil { + panic(fmt.Errorf("get %d free ports failed: %v", portCount, err)) + } + initialCluster := make(map[string]string) + for i := 0; i < count; i++ { + name := fmt.Sprintf("static-cluster-test-member-%03d", i) + peerUrl := fmt.Sprintf("http://localhost:%d", ports[(i*2)+1]) + initialCluster[name] = peerUrl + } + + for i := 0; i < count; i++ { + name := fmt.Sprintf("static-cluster-test-member-%03d", i) + opt := option.New() + opt.Name = name + opt.ClusterName = "test-static-sized-cluster" + opt.ClusterRole = "writer" + opt.ClusterRequestTimeout = "10s" + listenPort := ports[(i*2)+2] + advertisePort := ports[(i*2)+1] + + opt.APIAddr = fmt.Sprintf("localhost:%d", ports[0]) + opt.Cluster.ListenClientURLs = []string{fmt.Sprintf("http://localhost:%d", listenPort)} + opt.Cluster.AdvertiseClientURLs = opt.Cluster.ListenClientURLs + opt.Cluster.ListenPeerURLs = []string{fmt.Sprintf("http://localhost:%d", advertisePort)} + opt.Cluster.InitialAdvertisePeerURLs = opt.Cluster.ListenPeerURLs + opt.Cluster.InitialCluster = initialCluster + opt.HomeDir = filepath.Join(tempDir, name) + opt.DataDir = "data" + opt.LogDir = "log" + opt.MemberDir = "member" + opt.Debug = false + _, err = opt.Parse() // create directories + if err != nil { + panic(fmt.Errorf("parse option failed: %v", err)) + } + + id := uint64(i + 1) + + opts[i] = opt + members[i] = &member{ + ID: id, + Name: opt.Name, + PeerURL: opt.ClusterInitialAdvertisePeerURLs[0], + } + pbMembers[i] = &pb.Member{ + ID: id, + Name: opt.Name, + PeerURLs: []string{opt.ClusterInitialAdvertisePeerURLs[0]}, + ClientURLs: []string{opt.ClusterAdvertiseClientURLs[0]}, + } + env.InitServerDir(opts[i]) + } + sort.Sort(members) + tmp := members.copy() + if len(tmp) == 0 { + panic("members copy failed") + } + noexistMember := members.getByPeerURL("no-exist") + if noexistMember != nil { + panic("get a member not exist succ, should failed") + } + members.deleteByName("no-exist") + members.deleteByPeerURL("no-exist-purl") + return opts, members, pbMembers +} + func TestUpdateClusterMembers(t *testing.T) { opts, ms, pbMembers := mockMembers(9) From 5fb3ccf0d3b0e3ced601da949e542532ea0c0cf9 Mon Sep 17 00:00:00 2001 From: Samu Tamminen Date: Mon, 22 Nov 2021 11:26:08 +0800 Subject: [PATCH 05/17] format code --- pkg/cluster/cluster_test.go | 1 - pkg/cluster/config.go | 18 ++++++------ pkg/cluster/config_test.go | 8 +++--- pkg/cluster/member.go | 12 ++++---- pkg/option/option.go | 56 ++++++++++++++++++------------------- 5 files changed, 46 insertions(+), 49 deletions(-) diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 3dd1d666ee..f85267ba2b 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -131,7 +131,6 @@ func closeClusters(clusters []*cluster) { } } - func TestCluster(t *testing.T) { t.Run("start cluster dynamically", func(t *testing.T) { clusters := mockClusters(3) diff --git a/pkg/cluster/config.go b/pkg/cluster/config.go index e7486f2d7e..3ec3f31ed1 100644 --- a/pkg/cluster/config.go +++ b/pkg/cluster/config.go @@ -57,21 +57,20 @@ func CreateStaticClusterEtcdConfig(opt *option.Options, members *members) (*embe peerURLs []url.URL clientAdURLs []url.URL peerAdURLs []url.URL - ) - clientURLs,err := option.ParseURLs(opt.Cluster.ListenClientURLs) + clientURLs, err := option.ParseURLs(opt.Cluster.ListenClientURLs) if err != nil { return nil, err } - peerURLs,err = option.ParseURLs(opt.Cluster.ListenPeerURLs) + peerURLs, err = option.ParseURLs(opt.Cluster.ListenPeerURLs) if err != nil { return nil, err } - clientAdURLs,err = option.ParseURLs(opt.Cluster.AdvertiseClientURLs) + clientAdURLs, err = option.ParseURLs(opt.Cluster.AdvertiseClientURLs) if err != nil { return nil, err } - peerAdURLs,err = option.ParseURLs(opt.Cluster.InitialAdvertisePeerURLs) + peerAdURLs, err = option.ParseURLs(opt.Cluster.InitialAdvertisePeerURLs) if err != nil { return nil, err } @@ -103,7 +102,6 @@ func CreateStaticClusterEtcdConfig(opt *option.Options, members *members) (*embe return ec, nil } - // CreateEtcdConfig creates an embedded etcd config that starts the cluster by adding member by member. func CreateEtcdConfig(opt *option.Options, members *members) (*embed.Config, error) { ec := embed.NewConfig() @@ -114,19 +112,19 @@ func CreateEtcdConfig(opt *option.Options, members *members) (*embed.Config, err clientAdURLs []url.URL peerAdURLs []url.URL ) - clientURLs,err := option.ParseURLs(opt.ClusterListenClientURLs) + clientURLs, err := option.ParseURLs(opt.ClusterListenClientURLs) if err != nil { return nil, err } - peerURLs,err = option.ParseURLs(opt.ClusterListenPeerURLs) + peerURLs, err = option.ParseURLs(opt.ClusterListenPeerURLs) if err != nil { return nil, err } - clientAdURLs,err = option.ParseURLs(opt.ClusterAdvertiseClientURLs) + clientAdURLs, err = option.ParseURLs(opt.ClusterAdvertiseClientURLs) if err != nil { return nil, err } - peerAdURLs,err = option.ParseURLs(opt.ClusterInitialAdvertisePeerURLs) + peerAdURLs, err = option.ParseURLs(opt.ClusterInitialAdvertisePeerURLs) if err != nil { return nil, err } diff --git a/pkg/cluster/config_test.go b/pkg/cluster/config_test.go index 3ea3d40320..679a5d8605 100644 --- a/pkg/cluster/config_test.go +++ b/pkg/cluster/config_test.go @@ -19,8 +19,8 @@ package cluster import ( "fmt" - "testing" "strings" + "testing" "github.com/megaease/easegress/pkg/option" ) @@ -37,7 +37,7 @@ func TestCreateEtcdConfigFailures(t *testing.T) { testData[len(testData)-1].ClusterInitialAdvertisePeerURLs = []string{"::::::"} for i, opt := range testData { - t.Run(fmt.Sprintf("CreateEtcdConfig: options invalid url i=%d",i), func(t *testing.T) { + t.Run(fmt.Sprintf("CreateEtcdConfig: options invalid url i=%d", i), func(t *testing.T) { membersInstance, _ := newMembers(opt) _, err := CreateEtcdConfig(opt, membersInstance) if err == nil { @@ -60,7 +60,7 @@ func TestCreateEtcdConfigFailures(t *testing.T) { testData[len(testData)-1].Cluster.InitialAdvertisePeerURLs = []string{"::::::"} for i, opt := range testData { - t.Run(fmt.Sprintf("CreateStaticClusterEtcdConfig: options invalid url i=%d",i), func(t *testing.T) { + t.Run(fmt.Sprintf("CreateStaticClusterEtcdConfig: options invalid url i=%d", i), func(t *testing.T) { membersInstance, _ := newMembers(opt) _, err := CreateStaticClusterEtcdConfig(opt, membersInstance) if err == nil { @@ -68,7 +68,7 @@ func TestCreateEtcdConfigFailures(t *testing.T) { } if !strings.Contains(err.Error(), "missing protocol scheme") { t.Error("Error should contain missing protocol scheme") - } + } }) } } diff --git a/pkg/cluster/member.go b/pkg/cluster/member.go index 0b897a6d5c..cf35218746 100644 --- a/pkg/cluster/member.go +++ b/pkg/cluster/member.go @@ -111,12 +111,12 @@ func (m *members) initializeMembers(opt *option.Options) { // initializeStaticClusterMembers adds all members to ClusterMembers and KnownMembers. func (m *members) initializeStaticClusterMembers(opt *option.Options) { initMS := make(membersSlice, 0) - for name, peerURL := range opt.Cluster.InitialCluster { - initMS = append(initMS, &member{ - Name: name, - PeerURL: peerURL, - }) - } + for name, peerURL := range opt.Cluster.InitialCluster { + initMS = append(initMS, &member{ + Name: name, + PeerURL: peerURL, + }) + } m.ClusterMembers.update(initMS) m.KnownMembers.update(initMS) } diff --git a/pkg/option/option.go b/pkg/option/option.go index 324f0a4241..963b6061eb 100644 --- a/pkg/option/option.go +++ b/pkg/option/option.go @@ -37,11 +37,11 @@ import ( // ClusterOptions is the start-up options. type ClusterOptions struct { - ListenPeerURLs []string `yaml:"listen-peer-urls"` - ListenClientURLs []string `yaml:"listen-client-urls"` - AdvertiseClientURLs []string `yaml:"advertise-client-urls"` - InitialAdvertisePeerURLs []string `yaml:"initial-advertise-peer-urls"` - InitialCluster map[string]string `yaml:"initial-cluster"` + ListenPeerURLs []string `yaml:"listen-peer-urls"` + ListenClientURLs []string `yaml:"listen-client-urls"` + AdvertiseClientURLs []string `yaml:"advertise-client-urls"` + InitialAdvertisePeerURLs []string `yaml:"initial-advertise-peer-urls"` + InitialCluster map[string]string `yaml:"initial-cluster"` } // Options is the start-up options. @@ -61,23 +61,23 @@ type Options struct { // If a config file is specified, below command line flags will be ignored. // meta - Name string `yaml:"name" env:"EG_NAME"` - Labels map[string]string `yaml:"labels" env:"EG_LABELS"` - APIAddr string `yaml:"api-addr"` - Debug bool `yaml:"debug"` - DisableAccessLog bool `yaml:"disable-access-log"` - InitialObjectConfigFiles []string `yaml:"initial-object-config-files"` + Name string `yaml:"name" env:"EG_NAME"` + Labels map[string]string `yaml:"labels" env:"EG_LABELS"` + APIAddr string `yaml:"api-addr"` + Debug bool `yaml:"debug"` + DisableAccessLog bool `yaml:"disable-access-log"` + InitialObjectConfigFiles []string `yaml:"initial-object-config-files"` // cluster options - ClusterName string `yaml:"cluster-name"` - ClusterRole string `yaml:"cluster-role"` - ClusterRequestTimeout string `yaml:"cluster-request-timeout"` - ClusterListenClientURLs []string `yaml:"cluster-listen-client-urls"` - ClusterListenPeerURLs []string `yaml:"cluster-listen-peer-urls"` - ClusterAdvertiseClientURLs []string `yaml:"cluster-advertise-client-urls"` - ClusterInitialAdvertisePeerURLs []string `yaml:"cluster-initial-advertise-peer-urls"` - ClusterJoinURLs []string `yaml:"cluster-join-urls"` - Cluster ClusterOptions `yaml:"cluster"` + ClusterName string `yaml:"cluster-name"` + ClusterRole string `yaml:"cluster-role"` + ClusterRequestTimeout string `yaml:"cluster-request-timeout"` + ClusterListenClientURLs []string `yaml:"cluster-listen-client-urls"` + ClusterListenPeerURLs []string `yaml:"cluster-listen-peer-urls"` + ClusterAdvertiseClientURLs []string `yaml:"cluster-advertise-client-urls"` + ClusterInitialAdvertisePeerURLs []string `yaml:"cluster-initial-advertise-peer-urls"` + ClusterJoinURLs []string `yaml:"cluster-join-urls"` + Cluster ClusterOptions `yaml:"cluster"` // Path. HomeDir string `yaml:"home-dir"` @@ -103,7 +103,7 @@ func addClusterVars(opt *Options) { opt.flags.StringVar(&opt.ClusterName, "cluster-name", "eg-cluster-default-name", "Human-readable name for the new cluster, ignored while joining an existed cluster.") opt.flags.StringVar(&opt.ClusterRole, "cluster-role", "writer", "Cluster role for this member (reader, writer).") opt.flags.StringVar(&opt.ClusterRequestTimeout, "cluster-request-timeout", "10s", "Timeout to handle request in the cluster.") - + // Cluster connection configuration style 1 opt.flags.StringSliceVar(&opt.ClusterListenClientURLs, "cluster-listen-client-urls", []string{"http://localhost:2379"}, "List of URLs to listen on for cluster client traffic.") opt.flags.StringSliceVar(&opt.ClusterListenPeerURLs, "cluster-listen-peer-urls", []string{"http://localhost:2380"}, "List of URLs to listen on for cluster peer traffic.") @@ -293,21 +293,21 @@ func (opt *Options) validate() error { } case "writer": argumentsToValidate := map[string][]string{ - "cluster-listen-client-urls": opt.ClusterListenClientURLs, - "cluster-listen-peer-urls": opt.ClusterListenPeerURLs, - "cluster-advertise-client-urls": opt.ClusterAdvertiseClientURLs, + "cluster-listen-client-urls": opt.ClusterListenClientURLs, + "cluster-listen-peer-urls": opt.ClusterListenPeerURLs, + "cluster-advertise-client-urls": opt.ClusterAdvertiseClientURLs, "cluster-initial-advertise-peer-urls": opt.ClusterInitialAdvertisePeerURLs, } if opt.UseInitialCluster() { argumentsToValidate = map[string][]string{ - "listen-client-urls": opt.Cluster.ListenClientURLs, - "listen-peer-urls": opt.Cluster.ListenPeerURLs, - "advertise-client-urls": opt.Cluster.AdvertiseClientURLs, + "listen-client-urls": opt.Cluster.ListenClientURLs, + "listen-peer-urls": opt.Cluster.ListenPeerURLs, + "advertise-client-urls": opt.Cluster.AdvertiseClientURLs, "initial-advertise-peer-urls": opt.Cluster.InitialAdvertisePeerURLs, } initialClusterUrls := make([]string, 0, len(opt.Cluster.InitialCluster)) - for _, value := range opt.Cluster.InitialCluster { + for _, value := range opt.Cluster.InitialCluster { initialClusterUrls = append(initialClusterUrls, value) } if _, err := ParseURLs(initialClusterUrls); err != nil { From 41d447a8849c041ea2fbdb2dda263c9df84bc376 Mon Sep 17 00:00:00 2001 From: Samu Tamminen Date: Mon, 22 Nov 2021 16:17:42 +0800 Subject: [PATCH 06/17] rename port in examples --- example/config/cluster/cluster-1.yaml | 2 +- example/config/cluster/cluster-2.yaml | 2 +- example/config/cluster/cluster-3.yaml | 2 +- example/config/cluster/legacy-cluster-1.yaml | 2 +- example/config/cluster/legacy-cluster-2.yaml | 2 +- example/config/cluster/legacy-cluster-3.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/example/config/cluster/cluster-1.yaml b/example/config/cluster/cluster-1.yaml index bade277079..b5a917d968 100644 --- a/example/config/cluster/cluster-1.yaml +++ b/example/config/cluster/cluster-1.yaml @@ -1,7 +1,7 @@ name: writer-1 cluster-name: cluster-test cluster-role: writer -api-addr: localhost:2831 +api-addr: localhost:2381 data-dir: ./data wal-dir: "" cpu-profile-file: diff --git a/example/config/cluster/cluster-2.yaml b/example/config/cluster/cluster-2.yaml index 94ea238d95..87c2aa7acd 100644 --- a/example/config/cluster/cluster-2.yaml +++ b/example/config/cluster/cluster-2.yaml @@ -1,7 +1,7 @@ name: writer-2 cluster-name: cluster-test cluster-role: writer -api-addr: localhost:2831 +api-addr: localhost:2381 data-dir: ./data wal-dir: "" cpu-profile-file: diff --git a/example/config/cluster/cluster-3.yaml b/example/config/cluster/cluster-3.yaml index 74156053b9..35d606580d 100644 --- a/example/config/cluster/cluster-3.yaml +++ b/example/config/cluster/cluster-3.yaml @@ -1,7 +1,7 @@ name: writer-3 cluster-name: cluster-test cluster-role: writer -api-addr: localhost:2831 +api-addr: localhost:2381 data-dir: ./data wal-dir: "" cpu-profile-file: diff --git a/example/config/cluster/legacy-cluster-1.yaml b/example/config/cluster/legacy-cluster-1.yaml index 2e4035bf17..52ff8fa14f 100644 --- a/example/config/cluster/legacy-cluster-1.yaml +++ b/example/config/cluster/legacy-cluster-1.yaml @@ -1,7 +1,7 @@ name: writer-1 cluster-name: cluster-test cluster-role: writer -api-addr: localhost:2831 +api-addr: localhost:2381 data-dir: ./data wal-dir: "" cpu-profile-file: diff --git a/example/config/cluster/legacy-cluster-2.yaml b/example/config/cluster/legacy-cluster-2.yaml index ea70242349..a281e0fb18 100644 --- a/example/config/cluster/legacy-cluster-2.yaml +++ b/example/config/cluster/legacy-cluster-2.yaml @@ -1,7 +1,7 @@ name: writer-2 cluster-name: cluster-test cluster-role: writer -api-addr: localhost:2831 +api-addr: localhost:2381 data-dir: ./data wal-dir: "" cpu-profile-file: diff --git a/example/config/cluster/legacy-cluster-3.yaml b/example/config/cluster/legacy-cluster-3.yaml index 79103b9d07..5de899f476 100644 --- a/example/config/cluster/legacy-cluster-3.yaml +++ b/example/config/cluster/legacy-cluster-3.yaml @@ -1,7 +1,7 @@ name: writer-3 cluster-name: cluster-test cluster-role: writer -api-addr: localhost:2831 +api-addr: localhost:2381 data-dir: ./data wal-dir: "" cpu-profile-file: From 531bfe8dfafb847abd70abf4ab6c3f9c61fff430 Mon Sep 17 00:00:00 2001 From: Samu Tamminen Date: Mon, 22 Nov 2021 18:02:05 +0800 Subject: [PATCH 07/17] Support new cluster config in cluster maintenance and MQTTProxy object. --- pkg/cluster/cluster.go | 7 +++++-- pkg/cluster/config.go | 3 +++ pkg/cluster/member_test.go | 6 +++--- pkg/object/mqttproxy/mqttproxy.go | 7 ++++++- pkg/option/option.go | 4 +++- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 34f86e2e58..15144ce945 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -736,9 +736,12 @@ func (c *cluster) defrag() { defragInterval = defragFailedInterval logger.Errorf("defrag failed: get client failed: %v", err) } - + defragmentURL := c.opt.ClusterAdvertiseClientURLs[0] + if c.opt.UseInitialCluster() { + defragmentURL = c.opt.Cluster.AdvertiseClientURLs[0] + } // NOTICE: It needs longer time than normal ones. - _, err = client.Defragment(c.longRequestContext(), c.opt.ClusterAdvertiseClientURLs[0]) + _, err = client.Defragment(c.longRequestContext(), defragmentURL) if err != nil { defragInterval = defragFailedInterval logger.Errorf("defrag failed: %v", err) diff --git a/pkg/cluster/config.go b/pkg/cluster/config.go index 3ec3f31ed1..65ab5a281f 100644 --- a/pkg/cluster/config.go +++ b/pkg/cluster/config.go @@ -94,6 +94,9 @@ func CreateStaticClusterEtcdConfig(opt *option.Options, members *members) (*embe ec.LogOutputs = []string{common.NormalizeZapLogPath(filepath.Join(opt.AbsLogDir, logFilename))} ec.ClusterState = embed.ClusterStateFlagNew + if opt.Cluster.StateFlag == "existing" { + ec.ClusterState = embed.ClusterStateFlagExisting + } ec.InitialCluster = members.initCluster() logger.Infof("etcd config: init-cluster:%s cluster-state:%s force-new-cluster:%v", diff --git a/pkg/cluster/member_test.go b/pkg/cluster/member_test.go index 5feaa4b332..4a6e30ef2c 100644 --- a/pkg/cluster/member_test.go +++ b/pkg/cluster/member_test.go @@ -182,13 +182,13 @@ func mockStaticClusterMembers(count int) ([]*option.Options, membersSlice, []*pb members[i] = &member{ ID: id, Name: opt.Name, - PeerURL: opt.ClusterInitialAdvertisePeerURLs[0], + PeerURL: opt.Cluster.InitialAdvertisePeerURLs[0], } pbMembers[i] = &pb.Member{ ID: id, Name: opt.Name, - PeerURLs: []string{opt.ClusterInitialAdvertisePeerURLs[0]}, - ClientURLs: []string{opt.ClusterAdvertiseClientURLs[0]}, + PeerURLs: []string{opt.Cluster.InitialAdvertisePeerURLs[0]}, + ClientURLs: []string{opt.Cluster.AdvertiseClientURLs[0]}, } env.InitServerDir(opts[i]) } diff --git a/pkg/object/mqttproxy/mqttproxy.go b/pkg/object/mqttproxy/mqttproxy.go index 08e1f8fddb..afe8f0fdaf 100644 --- a/pkg/object/mqttproxy/mqttproxy.go +++ b/pkg/object/mqttproxy/mqttproxy.go @@ -106,8 +106,13 @@ func memberURLFunc(superSpec *supervisor.Spec) func(string, string) ([]string, e } if memberStatus.Options.Name != egName { egURLs := memberStatus.Options.ClusterInitialAdvertisePeerURLs + peerUrlVariableName := "ClusterInitialAdvertisePeerURLs" + if memberStatus.Options.UseInitialCluster() { + egURLs = memberStatus.Options.Cluster.InitialAdvertisePeerURLs + peerUrlVariableName = "Cluster.InitialAdvertisePeerURLs" + } if len(egURLs) == 0 { - return nil, fmt.Errorf("easegress %v has empty ClusterInitialAdvertisePeerURLs %v", memberStatus.Options.Name, egURLs) + return nil, fmt.Errorf("easegress %v has empty %v %v", memberStatus.Options.Name, peerUrlVariableName, egURLs) } egURL := egURLs[0] apiAddr := memberStatus.Options.APIAddr diff --git a/pkg/option/option.go b/pkg/option/option.go index 963b6061eb..2d251015a7 100644 --- a/pkg/option/option.go +++ b/pkg/option/option.go @@ -42,6 +42,7 @@ type ClusterOptions struct { AdvertiseClientURLs []string `yaml:"advertise-client-urls"` InitialAdvertisePeerURLs []string `yaml:"initial-advertise-peer-urls"` InitialCluster map[string]string `yaml:"initial-cluster"` + StateFlag string `yaml:"state-flag"` } // Options is the start-up options. @@ -119,6 +120,7 @@ func addClusterVars(opt *Options) { opt.flags.StringSliceVar(&opt.Cluster.InitialAdvertisePeerURLs, "initial-advertise-peer-urls", []string{"http://localhost:2380"}, "List of this member’s peer URLs to advertise to the rest of the cluster.") opt.flags.StringToStringVarP(&opt.Cluster.InitialCluster, "initial-cluster", "", nil, "List of (member name, URL) pairs that will form the cluster. E.g. writer-1=http://localhost:2380. When used, leave cluster-join-urls empty.") + opt.flags.StringVar(&opt.Cluster.StateFlag, "state-flag", "new", "Cluster state (new, existing)") } // New creates a default Options. @@ -288,7 +290,7 @@ func (opt *Options) validate() error { if opt.ForceNewCluster { return fmt.Errorf("reader got force-new-cluster") } - if len(opt.ClusterJoinURLs) == 0 { + if !opt.UseInitialCluster() && len(opt.ClusterJoinURLs) == 0 { return fmt.Errorf("reader got empty cluster-join-urls") } case "writer": From 387a17b39d938a9798335ff61ac314fa94e8252b Mon Sep 17 00:00:00 2001 From: Samu Tamminen Date: Tue, 23 Nov 2021 16:15:01 +0800 Subject: [PATCH 08/17] Remove logging and make option validation more readable --- pkg/option/option.go | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/pkg/option/option.go b/pkg/option/option.go index 2d251015a7..c0b3d6f1de 100644 --- a/pkg/option/option.go +++ b/pkg/option/option.go @@ -266,7 +266,6 @@ func ParseURLs(urlStrings []string) ([]url.URL, error) { for i, urlString := range urlStrings { parsedUrl, err := url.Parse(urlString) if err != nil { - fmt.Println(urlString) return nil, fmt.Errorf(" %s: %v", urlString, err) } urls[i] = *parsedUrl @@ -274,6 +273,25 @@ func ParseURLs(urlStrings []string) ([]url.URL, error) { return urls, nil } +// checkNoOverlappingArguments checks that only one of Cluster.InitialCluster and ClusterJoinURLs is defined. +func checkNoOverlappingArguments(opt *Options) error { + if !opt.UseInitialCluster() { + return nil + } + if len(opt.ClusterJoinURLs) == 0 { + return nil + } + hasYAMLConfig := len(opt.ConfigFile) > 0 + if hasYAMLConfig { + errorMsg := `cluster.initial-cluster and cluster-join-urls are both defined. ` + + `Please provide only one of them. cluster.initial-cluster is the recommended way.` + return fmt.Errorf(errorMsg) + } + errorMsg := `--initial-cluster and --cluster-join-urls are both defined. ` + + `Please provide only one of them. --initial-cluster is the recommended way.` + return fmt.Errorf(errorMsg) +} + func (opt *Options) validate() error { if opt.ClusterName == "" { return fmt.Errorf("empty cluster-name") @@ -294,6 +312,9 @@ func (opt *Options) validate() error { return fmt.Errorf("reader got empty cluster-join-urls") } case "writer": + if err := checkNoOverlappingArguments(opt); err != nil { + return err + } argumentsToValidate := map[string][]string{ "cluster-listen-client-urls": opt.ClusterListenClientURLs, "cluster-listen-peer-urls": opt.ClusterListenPeerURLs, @@ -315,15 +336,6 @@ func (opt *Options) validate() error { if _, err := ParseURLs(initialClusterUrls); err != nil { return fmt.Errorf("invalid initial-cluster: %v", err) } - if len(opt.ClusterJoinURLs) > 0 { - err := strings.Join(strings.Fields(` - %s and %s are both defined. - Please provide only one of them. %s is the recommended way.`), " ") - if len(opt.ConfigFile) == 0 { - return fmt.Errorf(err, "--initial-cluster", "--cluster-join-urls", "--initial-cluster") - } - return fmt.Errorf(err, "cluster.initial-cluster", "cluster-join-urls", "cluster.initial-cluster") - } } for arg, urls := range argumentsToValidate { if len(urls) == 0 { From c2ab192450ac0aba69ba3c8054c70a15efaa9f43 Mon Sep 17 00:00:00 2001 From: Samu Tamminen Date: Tue, 23 Nov 2021 16:19:50 +0800 Subject: [PATCH 09/17] format code --- pkg/option/option.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/option/option.go b/pkg/option/option.go index c0b3d6f1de..46519024d5 100644 --- a/pkg/option/option.go +++ b/pkg/option/option.go @@ -288,7 +288,7 @@ func checkNoOverlappingArguments(opt *Options) error { return fmt.Errorf(errorMsg) } errorMsg := `--initial-cluster and --cluster-join-urls are both defined. ` + - `Please provide only one of them. --initial-cluster is the recommended way.` + `Please provide only one of them. --initial-cluster is the recommended way.` return fmt.Errorf(errorMsg) } From 3e88aa89d39aa8045a10e0b71c407127222bfc34 Mon Sep 17 00:00:00 2001 From: Samu Tamminen Date: Wed, 24 Nov 2021 09:25:51 +0800 Subject: [PATCH 10/17] rename variables --- pkg/cluster/member_test.go | 4 ++-- pkg/object/mqttproxy/mqttproxy.go | 6 +++--- pkg/option/option.go | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/cluster/member_test.go b/pkg/cluster/member_test.go index 4a6e30ef2c..f2f708677e 100644 --- a/pkg/cluster/member_test.go +++ b/pkg/cluster/member_test.go @@ -146,8 +146,8 @@ func mockStaticClusterMembers(count int) ([]*option.Options, membersSlice, []*pb initialCluster := make(map[string]string) for i := 0; i < count; i++ { name := fmt.Sprintf("static-cluster-test-member-%03d", i) - peerUrl := fmt.Sprintf("http://localhost:%d", ports[(i*2)+1]) - initialCluster[name] = peerUrl + peerURL := fmt.Sprintf("http://localhost:%d", ports[(i*2)+1]) + initialCluster[name] = peerURL } for i := 0; i < count; i++ { diff --git a/pkg/object/mqttproxy/mqttproxy.go b/pkg/object/mqttproxy/mqttproxy.go index afe8f0fdaf..4f4b8a95a1 100644 --- a/pkg/object/mqttproxy/mqttproxy.go +++ b/pkg/object/mqttproxy/mqttproxy.go @@ -106,13 +106,13 @@ func memberURLFunc(superSpec *supervisor.Spec) func(string, string) ([]string, e } if memberStatus.Options.Name != egName { egURLs := memberStatus.Options.ClusterInitialAdvertisePeerURLs - peerUrlVariableName := "ClusterInitialAdvertisePeerURLs" + peerURLVariableName := "ClusterInitialAdvertisePeerURLs" if memberStatus.Options.UseInitialCluster() { egURLs = memberStatus.Options.Cluster.InitialAdvertisePeerURLs - peerUrlVariableName = "Cluster.InitialAdvertisePeerURLs" + peerURLVariableName = "Cluster.InitialAdvertisePeerURLs" } if len(egURLs) == 0 { - return nil, fmt.Errorf("easegress %v has empty %v %v", memberStatus.Options.Name, peerUrlVariableName, egURLs) + return nil, fmt.Errorf("easegress %v has empty %v %v", memberStatus.Options.Name, peerURLVariableName, egURLs) } egURL := egURLs[0] apiAddr := memberStatus.Options.APIAddr diff --git a/pkg/option/option.go b/pkg/option/option.go index 46519024d5..87efad0de3 100644 --- a/pkg/option/option.go +++ b/pkg/option/option.go @@ -264,11 +264,11 @@ func (opt *Options) adjust() { func ParseURLs(urlStrings []string) ([]url.URL, error) { urls := make([]url.URL, len(urlStrings)) for i, urlString := range urlStrings { - parsedUrl, err := url.Parse(urlString) + parsedURL, err := url.Parse(urlString) if err != nil { return nil, fmt.Errorf(" %s: %v", urlString, err) } - urls[i] = *parsedUrl + urls[i] = *parsedURL } return urls, nil } From beb28c3f945b5c9ff0f3bef9e2424ff1e85efcb9 Mon Sep 17 00:00:00 2001 From: Samu Tamminen Date: Thu, 25 Nov 2021 10:28:49 +0800 Subject: [PATCH 11/17] rename writer --> primary and reader --> secondary --- README.md | 4 +-- README.zh-CN.md | 4 +-- doc/benchmark.md | 2 +- doc/ingresscontroller.md | 2 +- example/config/cluster/cluster-1.yaml | 10 +++--- example/config/cluster/cluster-2.yaml | 10 +++--- example/config/cluster/cluster-3.yaml | 10 +++--- example/config/cluster/legacy-cluster-1.yaml | 4 +-- example/config/cluster/legacy-cluster-2.yaml | 4 +-- example/config/cluster/legacy-cluster-3.yaml | 4 +-- .../config/cluster/no-cluster-example.yaml | 4 +-- example/sbin/conf/config.yaml | 2 +- example/sbin/status.sh | 2 +- example/writer-001/conf/config.yaml | 2 +- example/writer-002/conf/config.yaml | 2 +- example/writer-003/conf/config.yaml | 2 +- pkg/cluster/cluster.go | 8 ++--- pkg/cluster/layout.go | 2 +- pkg/cluster/member.go | 4 +-- pkg/cluster/member_test.go | 4 +-- pkg/option/option.go | 32 ++++++++++++++----- 21 files changed, 67 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index de0c8da400..02f5a1467a 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ The architecture of Easegress: - Built-in [Open Zipkin](https://zipkin.io/) - [Open Tracing](https://opentracing.io/) for vendor-neutral APIs - **Observability** - - **Node:** role(leader, writer, reader), health or not, last heartbeat time, and so on + - **Node:** role(primary, secondary), raft leader status, healthy or not, last heartbeat time, and so on - **Traffic:** in multi-dimension: server and backend. - **Throughput:** total and error statistics of request count, TPS/m1, m5, m15, and error percent, etc. - **Latency:** p25, p50, p75, p95, p98, p99, p999. @@ -170,7 +170,7 @@ $ egctl member list name: eg-default-name labels: {} cluster-name: eg-cluster-default-name - cluster-role: writer + cluster-role: primary cluster-request-timeout: 10s cluster-listen-client-urls: - http://127.0.0.1:2379 diff --git a/README.zh-CN.md b/README.zh-CN.md index 26e0de449f..f4ccdce665 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -88,7 +88,7 @@ - 内置 [Open Zipkin](https://zipkin.io/) - [Open Tracing](https://opentracing.io/)，提供厂商中立的 API。 - **可观察性** - - **节点**：角色（Leader、Writer、Reader）、健康状态、最后一次心跳时间，等等。 + - **节点**：角色（primary、secondary）、是不是Leader，健康状态、最后一次心跳时间，等等。 - **多维度的服务器和后端流量数据** - **吞吐量**：请求数、TPS/m1、m5、m15 和错误百分比等。 - **延迟**：p25、p50、p75、p95、p98、p99、p999。 @@ -169,7 +169,7 @@ $ egctl member list name: eg-default-name labels: {} cluster-name: eg-cluster-default-name - cluster-role: writer + cluster-role: primary cluster-request-timeout: 10s cluster-listen-client-urls: - http://127.0.0.1:2379 diff --git a/doc/benchmark.md b/doc/benchmark.md index 0502de8a68..0ddaa67cdb 100644 --- a/doc/benchmark.md +++ b/doc/benchmark.md @@ -140,7 +140,7 @@ server { ``` yaml name: member-001 cluster-name: cluster-test -cluster-role: writer +cluster-role: primary cluster-client-url: http://127.0.0.1:2379 cluster-peer-url: http://127.0.0.1:2380 cluster-join-urls: diff --git a/doc/ingresscontroller.md b/doc/ingresscontroller.md index 890ff4a0be..0e0d5b49c3 100644 --- a/doc/ingresscontroller.md +++ b/doc/ingresscontroller.md @@ -103,7 +103,7 @@ spec: - |- echo name: $POD_NAME > /easegress-ingress/config.yaml && echo ' cluster-request-timeout: 10s - cluster-role: writer + cluster-role: primary api-addr: 0.0.0.0:2381 cluster-name: easegress-ingress-controller ' >> /easegress-ingress/config.yaml && echo ' diff --git a/example/config/cluster/cluster-1.yaml b/example/config/cluster/cluster-1.yaml index b5a917d968..350c7ee1ae 100644 --- a/example/config/cluster/cluster-1.yaml +++ b/example/config/cluster/cluster-1.yaml @@ -1,6 +1,6 @@ -name: writer-1 +name: primary-1 cluster-name: cluster-test -cluster-role: writer +cluster-role: primary api-addr: localhost:2381 data-dir: ./data wal-dir: "" @@ -18,6 +18,6 @@ cluster: initial-advertise-peer-urls: - http://127.0.0.1:2380 initial-cluster: - - writer-1: http://127.0.0.1:2380 - - writer-2: http://127.0.0.1:2378 - - writer-3: http://127.0.0.1:2376 + - primary-1: http://127.0.0.1:2380 + - primary-2: http://127.0.0.1:2378 + - primary-3: http://127.0.0.1:2376 diff --git a/example/config/cluster/cluster-2.yaml b/example/config/cluster/cluster-2.yaml index 87c2aa7acd..0ef5856de0 100644 --- a/example/config/cluster/cluster-2.yaml +++ b/example/config/cluster/cluster-2.yaml @@ -1,6 +1,6 @@ -name: writer-2 +name: primary-2 cluster-name: cluster-test -cluster-role: writer +cluster-role: primary api-addr: localhost:2381 data-dir: ./data wal-dir: "" @@ -18,6 +18,6 @@ cluster: initial-advertise-peer-urls: - http://127.0.0.1:2378 initial-cluster: - - writer-1: http://127.0.0.1:2380 - - writer-2: http://127.0.0.1:2378 - - writer-3: http://127.0.0.1:2376 + - primary-1: http://127.0.0.1:2380 + - primary-2: http://127.0.0.1:2378 + - primary-3: http://127.0.0.1:2376 diff --git a/example/config/cluster/cluster-3.yaml b/example/config/cluster/cluster-3.yaml index 35d606580d..5f028ea3f0 100644 --- a/example/config/cluster/cluster-3.yaml +++ b/example/config/cluster/cluster-3.yaml @@ -1,6 +1,6 @@ -name: writer-3 +name: primary-3 cluster-name: cluster-test -cluster-role: writer +cluster-role: primary api-addr: localhost:2381 data-dir: ./data wal-dir: "" @@ -18,6 +18,6 @@ cluster: initial-advertise-peer-urls: - http://127.0.0.1:2376 initial-cluster: - - writer-1: http://127.0.0.1:2380 - - writer-2: http://127.0.0.1:2378 - - writer-3: http://127.0.0.1:2376 + - primary-1: http://127.0.0.1:2380 + - primary-2: http://127.0.0.1:2378 + - primary-3: http://127.0.0.1:2376 diff --git a/example/config/cluster/legacy-cluster-1.yaml b/example/config/cluster/legacy-cluster-1.yaml index 52ff8fa14f..0d91323d35 100644 --- a/example/config/cluster/legacy-cluster-1.yaml +++ b/example/config/cluster/legacy-cluster-1.yaml @@ -1,6 +1,6 @@ -name: writer-1 +name: primary-1 cluster-name: cluster-test -cluster-role: writer +cluster-role: primary api-addr: localhost:2381 data-dir: ./data wal-dir: "" diff --git a/example/config/cluster/legacy-cluster-2.yaml b/example/config/cluster/legacy-cluster-2.yaml index a281e0fb18..35609512cd 100644 --- a/example/config/cluster/legacy-cluster-2.yaml +++ b/example/config/cluster/legacy-cluster-2.yaml @@ -1,6 +1,6 @@ -name: writer-2 +name: primary-2 cluster-name: cluster-test -cluster-role: writer +cluster-role: primary api-addr: localhost:2381 data-dir: ./data wal-dir: "" diff --git a/example/config/cluster/legacy-cluster-3.yaml b/example/config/cluster/legacy-cluster-3.yaml index 5de899f476..9d8096752f 100644 --- a/example/config/cluster/legacy-cluster-3.yaml +++ b/example/config/cluster/legacy-cluster-3.yaml @@ -1,6 +1,6 @@ -name: writer-3 +name: primary-3 cluster-name: cluster-test -cluster-role: writer +cluster-role: primary api-addr: localhost:2381 data-dir: ./data wal-dir: "" diff --git a/example/config/cluster/no-cluster-example.yaml b/example/config/cluster/no-cluster-example.yaml index 9ea04d1c3e..b1b540832a 100644 --- a/example/config/cluster/no-cluster-example.yaml +++ b/example/config/cluster/no-cluster-example.yaml @@ -1,6 +1,6 @@ -name: writer-n +name: primary-n cluster-name: cluster-test -cluster-role: writer +cluster-role: primary api-addr: 0.0.0.0:2831 data-dir: ./data wal-dir: "" diff --git a/example/sbin/conf/config.yaml b/example/sbin/conf/config.yaml index 67639c58b7..35d1f1fd87 100644 --- a/example/sbin/conf/config.yaml +++ b/example/sbin/conf/config.yaml @@ -1,6 +1,6 @@ name: member-001 cluster-name: cluster-test -cluster-role: writer +cluster-role: primary cluster-client-url: http://127.0.0.1:2379 cluster-peer-url: http://127.0.0.1:2380 cluster-join-urls: diff --git a/example/sbin/status.sh b/example/sbin/status.sh index 6e750503f3..c294bdd2de 100755 --- a/example/sbin/status.sh +++ b/example/sbin/status.sh @@ -62,7 +62,7 @@ NC='\033[0m' # No Color if [ -n "$cluster" ] && [ -n "$name" ] && [ -n "$role" ] && [ -n "$status" ] \ && [ -n "$peer" ] && [ -n "$client" ] && [ -n $api ] then - if [ $role = "writer" ] + if [ $role = "primary" ] then if [ ! -n "$etcd" ] then diff --git a/example/writer-001/conf/config.yaml b/example/writer-001/conf/config.yaml index a74b13adca..36d5269ad7 100644 --- a/example/writer-001/conf/config.yaml +++ b/example/writer-001/conf/config.yaml @@ -1,6 +1,6 @@ name: writer-001 cluster-name: cluster-test -cluster-role: writer +cluster-role: primary cluster-listen-client-urls: [http://127.0.0.1:12379] cluster-listen-peer-urls: [http://127.0.0.1:12380] cluster-advertise-client-urls: [http://127.0.0.1:12379] diff --git a/example/writer-002/conf/config.yaml b/example/writer-002/conf/config.yaml index 8426839f69..2bd78ed948 100644 --- a/example/writer-002/conf/config.yaml +++ b/example/writer-002/conf/config.yaml @@ -1,6 +1,6 @@ name: writer-002 cluster-name: cluster-test -cluster-role: writer +cluster-role: primary cluster-listen-client-urls: [http://127.0.0.1:22379] cluster-listen-peer-urls: [http://127.0.0.1:22380] cluster-advertise-client-urls: [http://127.0.0.1:22379] diff --git a/example/writer-003/conf/config.yaml b/example/writer-003/conf/config.yaml index 65debae296..65de519b40 100644 --- a/example/writer-003/conf/config.yaml +++ b/example/writer-003/conf/config.yaml @@ -1,6 +1,6 @@ name: writer-003 cluster-name: cluster-test -cluster-role: writer +cluster-role: primary cluster-listen-client-urls: [http://127.0.0.1:32379] cluster-listen-peer-urls: [http://127.0.0.1:32380] cluster-advertise-client-urls: [http://127.0.0.1:32379] diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 15144ce945..b548b3cfd5 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -68,7 +68,7 @@ type ( LastDefragTime string `yaml:"lastDefragTime,omitempty"` - // Etcd is non-nil only it is a writer. + // Etcd is non-nil only if it's cluster status is primary. Etcd *EtcdStatus `yaml:"etcd,omitempty"` } @@ -212,7 +212,7 @@ func (c *cluster) run() { logger.Infof("cluster is ready") - if c.opt.ClusterRole == "writer" { + if c.opt.ClusterRole == "primary" { go c.defrag() } @@ -220,7 +220,7 @@ func (c *cluster) run() { } func (c *cluster) getReady() error { - if c.opt.ClusterRole == "reader" { + if c.opt.ClusterRole == "secondary" { _, err := c.getClient() if err != nil { return err @@ -761,7 +761,7 @@ func (c *cluster) syncStatus() error { Options: *c.opt, } - if c.opt.ClusterRole == "writer" { + if c.opt.ClusterRole == "primary" { server, err := c.getServer() if err != nil { return err diff --git a/pkg/cluster/layout.go b/pkg/cluster/layout.go index 57659c3225..c4d213950e 100644 --- a/pkg/cluster/layout.go +++ b/pkg/cluster/layout.go @@ -36,7 +36,7 @@ const ( wasmDataPrefixFormat = "/wasm/data/%s/%s/" // the cluster name of this eg group will be registered under this path in etcd - // any new member(reader or writer ) will be rejected if it is configured a different cluster name + // any new member(primary or secondary ) will be rejected if it is configured a different cluster name clusterNameKey = "/eg/cluster/name" ) diff --git a/pkg/cluster/member.go b/pkg/cluster/member.go index cf35218746..cd4985e168 100644 --- a/pkg/cluster/member.go +++ b/pkg/cluster/member.go @@ -88,7 +88,7 @@ func newMembers(opt *option.Options) (*members, error) { // initializeMembers adds first member to ClusterMembers and all members to KnownMembers. func (m *members) initializeMembers(opt *option.Options) { initMS := make(membersSlice, 0) - if opt.ClusterRole == "writer" && len(opt.ClusterInitialAdvertisePeerURLs) != 0 { + if opt.ClusterRole == "primary" && len(opt.ClusterInitialAdvertisePeerURLs) != 0 { // Cluster is started member by member --> start with cluster of size 1 initMS = append(initMS, &member{ Name: opt.Name, @@ -196,7 +196,7 @@ func (m *members) _self() *member { return s } - if m.opt.ClusterRole == "writer" { + if m.opt.ClusterRole == "primary" { logger.Errorf("BUG: can't get self from cluster members: %s "+ "knownMembers: %s", m.ClusterMembers, m.KnownMembers) } diff --git a/pkg/cluster/member_test.go b/pkg/cluster/member_test.go index f2f708677e..ece507ba9c 100644 --- a/pkg/cluster/member_test.go +++ b/pkg/cluster/member_test.go @@ -70,7 +70,7 @@ func mockTestOpt() *option.Options { opt := option.New() opt.Name = name opt.ClusterName = "test-cluster" - opt.ClusterRole = "writer" + opt.ClusterRole = "primary" opt.ClusterRequestTimeout = "10s" opt.ClusterListenClientURLs = []string{fmt.Sprintf("http://localhost:%d", ports[0])} opt.ClusterAdvertiseClientURLs = opt.ClusterListenClientURLs @@ -155,7 +155,7 @@ func mockStaticClusterMembers(count int) ([]*option.Options, membersSlice, []*pb opt := option.New() opt.Name = name opt.ClusterName = "test-static-sized-cluster" - opt.ClusterRole = "writer" + opt.ClusterRole = "primary" opt.ClusterRequestTimeout = "10s" listenPort := ports[(i*2)+2] advertisePort := ports[(i*2)+1] diff --git a/pkg/option/option.go b/pkg/option/option.go index 87efad0de3..2e4a7b2fb3 100644 --- a/pkg/option/option.go +++ b/pkg/option/option.go @@ -102,7 +102,7 @@ type Options struct { // addClusterVars introduces cluster arguments. func addClusterVars(opt *Options) { opt.flags.StringVar(&opt.ClusterName, "cluster-name", "eg-cluster-default-name", "Human-readable name for the new cluster, ignored while joining an existed cluster.") - opt.flags.StringVar(&opt.ClusterRole, "cluster-role", "writer", "Cluster role for this member (reader, writer).") + opt.flags.StringVar(&opt.ClusterRole, "cluster-role", "primary", "Cluster role for this member (primary, secondary).") opt.flags.StringVar(&opt.ClusterRequestTimeout, "cluster-request-timeout", "10s", "Timeout to handle request in the cluster.") // Cluster connection configuration style 1 @@ -119,7 +119,7 @@ func addClusterVars(opt *Options) { opt.flags.StringSliceVar(&opt.Cluster.AdvertiseClientURLs, "advertise-client-urls", []string{"http://localhost:2379"}, "List of this member’s client URLs to advertise to the rest of the cluster.") opt.flags.StringSliceVar(&opt.Cluster.InitialAdvertisePeerURLs, "initial-advertise-peer-urls", []string{"http://localhost:2380"}, "List of this member’s peer URLs to advertise to the rest of the cluster.") opt.flags.StringToStringVarP(&opt.Cluster.InitialCluster, "initial-cluster", "", nil, - "List of (member name, URL) pairs that will form the cluster. E.g. writer-1=http://localhost:2380. When used, leave cluster-join-urls empty.") + "List of (member name, URL) pairs that will form the cluster. E.g. primary-1=http://localhost:2380. When used, leave cluster-join-urls empty.") opt.flags.StringVar(&opt.Cluster.StateFlag, "state-flag", "new", "Cluster state (new, existing)") } @@ -167,6 +167,21 @@ func (opt *Options) UseInitialCluster() bool { return len(opt.Cluster.InitialCluster) > 0 } +// renameLegacyClusterRoles renames legacy writer/reader --> primary/secondary and raises warning. +func (opt *Options) renameLegacyClusterRoles() { + warning := "Cluster roles writer/reader are deprecated. \n" + + "Renamed cluster role '%s' to '%s'. Please use primary/secondary instead. \n" + fmtLogger := fmt.Printf // Importing logger here is a import cycle, so use fmt instead. + if opt.ClusterRole == "writer" { + opt.ClusterRole = "primary" + fmtLogger(warning, "writer", "primary") + } + if opt.ClusterRole == "reader" { + opt.ClusterRole = "secondary" + fmtLogger(warning, "reader", "secondary") + } +} + // Parse parses all arguments, returns normal message without error if --help/--version set. func (opt *Options) Parse() (string, error) { err := opt.flags.Parse(os.Args[1:]) @@ -212,6 +227,7 @@ func (opt *Options) Parse() (string, error) { c.TagName = "yaml" }) + opt.renameLegacyClusterRoles() err = opt.validate() if err != nil { return "", err @@ -240,7 +256,7 @@ func (opt *Options) Parse() (string, error) { // adjust adjusts the options to handle conflict // between user's config and internal component. func (opt *Options) adjust() { - if opt.ClusterRole != "writer" || opt.UseInitialCluster() { + if opt.ClusterRole != "primary" || opt.UseInitialCluster() { return } if len(opt.ClusterJoinURLs) == 0 { @@ -304,14 +320,14 @@ func (opt *Options) validate() error { } } switch opt.ClusterRole { - case "reader": + case "secondary": if opt.ForceNewCluster { - return fmt.Errorf("reader got force-new-cluster") + return fmt.Errorf("secondary got force-new-cluster") } if !opt.UseInitialCluster() && len(opt.ClusterJoinURLs) == 0 { - return fmt.Errorf("reader got empty cluster-join-urls") + return fmt.Errorf("secondary got empty cluster-join-urls") } - case "writer": + case "primary": if err := checkNoOverlappingArguments(opt); err != nil { return err } @@ -346,7 +362,7 @@ func (opt *Options) validate() error { } } default: - return fmt.Errorf("invalid cluster-role(support writer, reader)") + return fmt.Errorf("invalid cluster-role. Supported roles are primary/secondary.") } _, err := time.ParseDuration(opt.ClusterRequestTimeout) From 69e62aedc32766d0316f21b90b18f022650445a8 Mon Sep 17 00:00:00 2001 From: Samu Tamminen Date: Thu, 25 Nov 2021 17:45:28 +0800 Subject: [PATCH 12/17] do not use cluster/members.go for new config and separate primary and secondary URLs. --- pkg/cluster/cluster.go | 63 +++++++++++++++++++++++++++++--------- pkg/cluster/config.go | 4 +-- pkg/cluster/config_test.go | 3 +- pkg/cluster/member.go | 36 +++++----------------- pkg/option/option.go | 59 +++++++++++++++++++++++++++++++++-- 5 files changed, 116 insertions(+), 49 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index b548b3cfd5..5bb403e14a 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -26,6 +26,7 @@ import ( "sync" "time" + pb "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" "go.etcd.io/etcd/server/v3/embed" @@ -144,15 +145,21 @@ func New(opt *option.Options) (Cluster, error) { return nil, fmt.Errorf("invalid cluster request timeout: %v", err) } - members, err := newMembers(opt) - if err != nil { - return nil, fmt.Errorf("new members failed: %v", err) + // Member file，members.ClusterMembers and members.KnownMembers will be deprecated in the future. + // When the new configuration way (cluster.initial-cluster or cluster.primary-listen-peer-urls) is used, let's not create member + // instance but let's read member information from pkg/option/options.go's Options.ClusterOptions directly. + var membersFile *members + if len(opt.GetPeerURLs()) == 0 { + membersFile, err = newMembers(opt) + if err != nil { + return nil, fmt.Errorf("new members failed: %v", err) + } } c := &cluster{ opt: opt, requestTimeout: requestTimeout, - members: members, + members: membersFile, done: make(chan struct{}), } @@ -241,7 +248,7 @@ func (c *cluster) getReady() error { return nil } - if !c.opt.ForceNewCluster && c.members.knownMembersLen() > 1 && !c.opt.UseInitialCluster() { + if !c.opt.UseInitialCluster() && !c.opt.ForceNewCluster && c.members != nil && c.members.knownMembersLen() > 1 { client, _ := c.getClient() if client != nil { err := c.addSelfToCluster() @@ -382,9 +389,14 @@ func (c *cluster) getClient() (*clientv3.Client, error) { return c.client, nil } - endpoints := c.members.knownPeerURLs() - if c.opt.ForceNewCluster { - endpoints = []string{c.members.self().PeerURL} + var endpoints []string + if c.members == nil { + endpoints = c.opt.GetPeerURLs() + } else { + endpoints = c.members.knownPeerURLs() + if c.opt.ForceNewCluster { + endpoints = []string{c.members.self().PeerURL} + } } logger.Infof("client connect with endpoints: %v", endpoints) client, err := clientv3.New(clientv3.Config{ @@ -632,12 +644,14 @@ func (c *cluster) startServer() (done, timeout chan struct{}, err error) { close(done) return done, timeout, nil } - createConfig := CreateEtcdConfig + var ( + etcdConfig *embed.Config + ) if c.opt.UseInitialCluster() { - createConfig = CreateStaticClusterEtcdConfig + etcdConfig, err = CreateStaticClusterEtcdConfig(c.opt) + } else { + etcdConfig, err = CreateEtcdConfig(c.opt, c.members) } - - etcdConfig, err := createConfig(c.opt, c.members) if err != nil { return nil, nil, err } @@ -800,11 +814,32 @@ func (c *cluster) updateMembers() error { return err } - c.members.updateClusterMembers(resp.Members) - + if c.opt.UseInitialCluster() { + c.UpdatePeerURLs(resp.Members) + } else { + c.members.updateClusterMembers(resp.Members) + } return nil } +// UpdatePeerURLs updates peerURLs according the cluster role. +func (c *cluster) UpdatePeerURLs(pbMembers []*pb.Member) { + primaryMembers := pbMembersToMembersSlice(pbMembers) + if c.opt.ClusterRole == "secondary" { + newPeerURLs := make([]string, 0) + for _, member := range primaryMembers { + newPeerURLs = append(newPeerURLs, member.PeerURL) + } + c.opt.SetClusterPrimaryListenPeerURLs(newPeerURLs) + } else { + newInitialCluster := make(map[string]string) + for _, member := range primaryMembers { + newInitialCluster[member.Name] = member.PeerURL + } + c.opt.SetInitialCluster(newInitialCluster) + } +} + func (c *cluster) PurgeMember(memberName string) error { client, err := c.getClient() if err != nil { diff --git a/pkg/cluster/config.go b/pkg/cluster/config.go index 65ab5a281f..bde108ae92 100644 --- a/pkg/cluster/config.go +++ b/pkg/cluster/config.go @@ -49,7 +49,7 @@ var ( // CreateStaticClusterEtcdConfig creates an embedded etcd config for static sized cluster, // listing all cluster members for etcd's initial-cluster argument. -func CreateStaticClusterEtcdConfig(opt *option.Options, members *members) (*embed.Config, error) { +func CreateStaticClusterEtcdConfig(opt *option.Options) (*embed.Config, error) { ec := embed.NewConfig() var ( @@ -97,7 +97,7 @@ func CreateStaticClusterEtcdConfig(opt *option.Options, members *members) (*embe if opt.Cluster.StateFlag == "existing" { ec.ClusterState = embed.ClusterStateFlagExisting } - ec.InitialCluster = members.initCluster() + ec.InitialCluster = opt.InitialClusterToString() logger.Infof("etcd config: init-cluster:%s cluster-state:%s force-new-cluster:%v", ec.InitialCluster, ec.ClusterState, ec.ForceNewCluster) diff --git a/pkg/cluster/config_test.go b/pkg/cluster/config_test.go index 679a5d8605..3256a2428e 100644 --- a/pkg/cluster/config_test.go +++ b/pkg/cluster/config_test.go @@ -61,8 +61,7 @@ func TestCreateEtcdConfigFailures(t *testing.T) { for i, opt := range testData { t.Run(fmt.Sprintf("CreateStaticClusterEtcdConfig: options invalid url i=%d", i), func(t *testing.T) { - membersInstance, _ := newMembers(opt) - _, err := CreateStaticClusterEtcdConfig(opt, membersInstance) + _, err := CreateStaticClusterEtcdConfig(opt) if err == nil { t.Error("There should be an error") } diff --git a/pkg/cluster/member.go b/pkg/cluster/member.go index cd4985e168..0814d40315 100644 --- a/pkg/cluster/member.go +++ b/pkg/cluster/member.go @@ -72,16 +72,11 @@ func newMembers(opt *option.Options) (*members, error) { ClusterMembers: newMemberSlices(), KnownMembers: newMemberSlices(), } - if opt.UseInitialCluster() { - m.initializeStaticClusterMembers(opt) - } else { - m.initializeMembers(opt) - err := m.load() - if err != nil { - return nil, err - } + m.initializeMembers(opt) + err := m.load() + if err != nil { + return nil, err } - return m, nil } @@ -108,19 +103,6 @@ func (m *members) initializeMembers(opt *option.Options) { m.KnownMembers.update(initMS) } -// initializeStaticClusterMembers adds all members to ClusterMembers and KnownMembers. -func (m *members) initializeStaticClusterMembers(opt *option.Options) { - initMS := make(membersSlice, 0) - for name, peerURL := range opt.Cluster.InitialCluster { - initMS = append(initMS, &member{ - Name: name, - PeerURL: peerURL, - }) - } - m.ClusterMembers.update(initMS) - m.KnownMembers.update(initMS) -} - func (m *members) fileExist() bool { _, err := os.Stat(m.file) return !os.IsNotExist(err) @@ -245,12 +227,10 @@ func (m *members) updateClusterMembers(pbMembers []*pb.Member) { // NOTE: KnownMembers store members as many as possible m.KnownMembers.update(*m.ClusterMembers) - if !m.opt.UseInitialCluster() { - // When cluster is initialized member by member, persist KnownMembers and ClusterMembers - // to disk for failure recovery. If a member fails for any reason before it has been added - // to cluster, the persisted file can be used to continue initialization. - m.store() - } + // When cluster is initialized member by member, persist KnownMembers and ClusterMembers + // to disk for failure recovery. If a member fails for any reason before it has been added + // to cluster, the persisted file can be used to continue initialization. + m.store() } func (m *members) knownMembersLen() int { diff --git a/pkg/option/option.go b/pkg/option/option.go index 2e4a7b2fb3..fd2b4c3a5e 100644 --- a/pkg/option/option.go +++ b/pkg/option/option.go @@ -24,6 +24,7 @@ import ( "os" "path/filepath" "strings" + "sync" "time" "github.com/mitchellh/mapstructure" @@ -37,12 +38,15 @@ import ( // ClusterOptions is the start-up options. type ClusterOptions struct { + // Primary members define following URLs to form a cluster. ListenPeerURLs []string `yaml:"listen-peer-urls"` ListenClientURLs []string `yaml:"listen-client-urls"` AdvertiseClientURLs []string `yaml:"advertise-client-urls"` InitialAdvertisePeerURLs []string `yaml:"initial-advertise-peer-urls"` InitialCluster map[string]string `yaml:"initial-cluster"` StateFlag string `yaml:"state-flag"` + // Secondary members define URLs to connect to cluster formed by primary members. + PrimaryListenPeerURLs []string `yaml:"primary-listen-peer-urls"` } // Options is the start-up options. @@ -97,6 +101,9 @@ type Options struct { AbsWALDir string `yaml:"-"` AbsLogDir string `yaml:"-"` AbsMemberDir string `yaml:"-"` + + // For Options in-memory updates. Updates are not persisted. + mutex sync.Mutex } // addClusterVars introduces cluster arguments. @@ -121,6 +128,10 @@ func addClusterVars(opt *Options) { opt.flags.StringToStringVarP(&opt.Cluster.InitialCluster, "initial-cluster", "", nil, "List of (member name, URL) pairs that will form the cluster. E.g. primary-1=http://localhost:2380. When used, leave cluster-join-urls empty.") opt.flags.StringVar(&opt.Cluster.StateFlag, "state-flag", "new", "Cluster state (new, existing)") + opt.flags.StringSliceVar(&opt.Cluster.PrimaryListenPeerURLs, + "primary-listen-peer-urls", + []string{"http://localhost:2380"}, + "List of peer URLs of primary members. Define this only, when cluster-role is secondary.") } // New creates a default Options. @@ -128,6 +139,7 @@ func New() *Options { opt := &Options{ flags: pflag.NewFlagSet(os.Args[0], pflag.ContinueOnError), viper: viper.New(), + mutex: sync.Mutex{}, } opt.flags.BoolVarP(&opt.ShowVersion, "version", "v", false, "Print the version and exit.") @@ -324,8 +336,8 @@ func (opt *Options) validate() error { if opt.ForceNewCluster { return fmt.Errorf("secondary got force-new-cluster") } - if !opt.UseInitialCluster() && len(opt.ClusterJoinURLs) == 0 { - return fmt.Errorf("secondary got empty cluster-join-urls") + if len(opt.Cluster.PrimaryListenPeerURLs) == 0 && len(opt.ClusterJoinURLs) == 0 { + return fmt.Errorf("secondary got empty cluster.primary-listen-peer-urls and cluster-join-urls entries") } case "primary": if err := checkNoOverlappingArguments(opt); err != nil { @@ -362,7 +374,7 @@ func (opt *Options) validate() error { } } default: - return fmt.Errorf("invalid cluster-role. Supported roles are primary/secondary.") + return fmt.Errorf("invalid cluster-role: supported roles are primary/secondary") } _, err := time.ParseDuration(opt.ClusterRequestTimeout) @@ -446,6 +458,47 @@ func (opt *Options) prepare() error { return nil } +// InitialClusterToString returns initial clusters string representation. +func (opt *Options) InitialClusterToString() string { + ss := make([]string, 0) + for name, peerURL := range opt.Cluster.InitialCluster { + ss = append(ss, fmt.Sprintf("%s=%s", name, peerURL)) + } + return strings.Join(ss, ",") +} + +// GetPeerURLs returns URLs listed in cluster.initial-cluster for primary (a.k.a writer) and +// for secondary (a.k.a reader) the ones listed in cluster.primary-listen-peer-url. +func (opt *Options) GetPeerURLs() []string { + opt.mutex.Lock() + defer opt.mutex.Unlock() + + if opt.ClusterRole == "secondary" { + return opt.Cluster.PrimaryListenPeerURLs + } + peerURLs := make([]string, 0) + for _, peerURL := range opt.Cluster.InitialCluster { + peerURLs = append(peerURLs, peerURL) + } + return peerURLs +} + +// SetClusterPrimaryListenPeerURLs updates secondary role's peerURLs. +func (opt *Options) SetClusterPrimaryListenPeerURLs(peerURLs []string) { + opt.mutex.Lock() + defer opt.mutex.Unlock() + + opt.Cluster.PrimaryListenPeerURLs = peerURLs +} + +// SetInitialCluster updates primary role's initial cluster. +func (opt *Options) SetInitialCluster(peerURLs map[string]string) { + opt.mutex.Lock() + defer opt.mutex.Unlock() + + opt.Cluster.InitialCluster = peerURLs +} + func generateMemberName(apiAddr string) (string, error) { hostname, err := os.Hostname() if err != nil { From 3e1e338509c2e678f776c07a0f5543c25472c4b1 Mon Sep 17 00:00:00 2001 From: Samu Tamminen Date: Thu, 25 Nov 2021 18:03:50 +0800 Subject: [PATCH 13/17] update options when no member file --- example/config/cluster/cluster-4.yaml | 13 +++++++++++++ pkg/cluster/cluster.go | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 example/config/cluster/cluster-4.yaml diff --git a/example/config/cluster/cluster-4.yaml b/example/config/cluster/cluster-4.yaml new file mode 100644 index 0000000000..70096572ad --- /dev/null +++ b/example/config/cluster/cluster-4.yaml @@ -0,0 +1,13 @@ +name: secondary-1 +cluster-name: cluster-test +cluster-role: secondary +api-addr: localhost:2381 +data-dir: ./data +wal-dir: "" +cpu-profile-file: +memory-profile-file: +log-dir: ./log +debug: false +cluster: + primary-listen-peer-urls: + - http://127.0.0.1:2380 diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 5bb403e14a..ffb12e68a6 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -814,7 +814,7 @@ func (c *cluster) updateMembers() error { return err } - if c.opt.UseInitialCluster() { + if c.members == nil { c.UpdatePeerURLs(resp.Members) } else { c.members.updateClusterMembers(resp.Members) From 5eba5a7990a7231926ad68a0fb43f2df4edc0ef9 Mon Sep 17 00:00:00 2001 From: Samu Tamminen Date: Fri, 26 Nov 2021 14:31:45 +0800 Subject: [PATCH 14/17] rename example folders and update configs --- build/test/httpserver_test.sh | 18 ++++++++--------- example/README.md | 10 +++++----- example/check_cluster_status.sh | 6 +++--- example/clean_cluster.sh | 2 +- example/create_objects.sh | 2 +- example/delete_objects.sh | 2 +- example/primary-001/conf/config.yaml | 19 ++++++++++++++++++ example/{reader-004 => primary-001}/egctl.sh | 0 example/{reader-004 => primary-001}/start.sh | 0 example/{reader-004 => primary-001}/status.sh | 0 example/{reader-004 => primary-001}/stop.sh | 0 example/primary-002/conf/config.yaml | 20 +++++++++++++++++++ example/{reader-005 => primary-002}/egctl.sh | 0 example/{reader-005 => primary-002}/start.sh | 0 example/{reader-005 => primary-002}/status.sh | 0 example/{reader-005 => primary-002}/stop.sh | 0 example/primary-003/conf/config.yaml | 20 +++++++++++++++++++ example/{writer-001 => primary-003}/egctl.sh | 0 example/{writer-001 => primary-003}/start.sh | 0 example/{writer-001 => primary-003}/status.sh | 0 example/{writer-001 => primary-003}/stop.sh | 0 example/reader-004/README.md | 1 - example/reader-004/bin | 1 - example/reader-004/conf/config.yaml | 16 --------------- example/reader-005/README.md | 1 - example/reader-005/bin | 1 - example/reader-005/conf/config.yaml | 16 --------------- example/sbin/stop.sh | 4 ++-- example/secondary-004/conf/config.yaml | 14 +++++++++++++ .../{writer-002 => secondary-004}/egctl.sh | 0 .../{writer-002 => secondary-004}/start.sh | 0 .../{writer-002 => secondary-004}/status.sh | 0 example/{writer-002 => secondary-004}/stop.sh | 0 example/secondary-005/conf/config.yaml | 14 +++++++++++++ .../{writer-003 => secondary-005}/egctl.sh | 0 .../{writer-003 => secondary-005}/start.sh | 0 .../{writer-003 => secondary-005}/status.sh | 0 example/{writer-003 => secondary-005}/stop.sh | 0 example/start_cluster.sh | 2 +- example/stop_cluster.sh | 2 +- example/update_objects.sh | 2 +- example/writer-001/bin | 1 - example/writer-001/conf/config.yaml | 16 --------------- example/writer-002/README.md | 1 - example/writer-002/bin | 1 - example/writer-002/conf/config.yaml | 16 --------------- example/writer-003/README.md | 1 - example/writer-003/bin | 1 - example/writer-003/conf/config.yaml | 16 --------------- 49 files changed, 112 insertions(+), 114 deletions(-) create mode 100644 example/primary-001/conf/config.yaml rename example/{reader-004 => primary-001}/egctl.sh (100%) rename example/{reader-004 => primary-001}/start.sh (100%) rename example/{reader-004 => primary-001}/status.sh (100%) rename example/{reader-004 => primary-001}/stop.sh (100%) create mode 100644 example/primary-002/conf/config.yaml rename example/{reader-005 => primary-002}/egctl.sh (100%) rename example/{reader-005 => primary-002}/start.sh (100%) rename example/{reader-005 => primary-002}/status.sh (100%) rename example/{reader-005 => primary-002}/stop.sh (100%) create mode 100644 example/primary-003/conf/config.yaml rename example/{writer-001 => primary-003}/egctl.sh (100%) rename example/{writer-001 => primary-003}/start.sh (100%) rename example/{writer-001 => primary-003}/status.sh (100%) rename example/{writer-001 => primary-003}/stop.sh (100%) delete mode 120000 example/reader-004/README.md delete mode 120000 example/reader-004/bin delete mode 100644 example/reader-004/conf/config.yaml delete mode 120000 example/reader-005/README.md delete mode 120000 example/reader-005/bin delete mode 100644 example/reader-005/conf/config.yaml create mode 100644 example/secondary-004/conf/config.yaml rename example/{writer-002 => secondary-004}/egctl.sh (100%) rename example/{writer-002 => secondary-004}/start.sh (100%) rename example/{writer-002 => secondary-004}/status.sh (100%) rename example/{writer-002 => secondary-004}/stop.sh (100%) create mode 100644 example/secondary-005/conf/config.yaml rename example/{writer-003 => secondary-005}/egctl.sh (100%) rename example/{writer-003 => secondary-005}/start.sh (100%) rename example/{writer-003 => secondary-005}/status.sh (100%) rename example/{writer-003 => secondary-005}/stop.sh (100%) delete mode 120000 example/writer-001/bin delete mode 100644 example/writer-001/conf/config.yaml delete mode 120000 example/writer-002/README.md delete mode 120000 example/writer-002/bin delete mode 100644 example/writer-002/conf/config.yaml delete mode 120000 example/writer-003/README.md delete mode 120000 example/writer-003/bin delete mode 100644 example/writer-003/conf/config.yaml diff --git a/build/test/httpserver_test.sh b/build/test/httpserver_test.sh index beab38050c..1fecf110df 100755 --- a/build/test/httpserver_test.sh +++ b/build/test/httpserver_test.sh @@ -26,10 +26,10 @@ set -e SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" pushd $SCRIPTPATH"/../../example" > /dev/null EXAMPLEDIR="$SCRIPTPATH"/../../example -WRITER01DIR=$EXAMPLEDIR"/writer-001" +PRIMARY01DIR=$EXAMPLEDIR"/primary-001" # target file related define. -server="writer-001/bin/easegress-server" +server="primary-001/bin/easegress-server" backend="$EXAMPLEDIR/backend-service/echo/echo.go" httpsvr_port=10080 echo_port=9095 @@ -40,7 +40,7 @@ COLOR_NONE='\033[0m' COLOR_INFO='\033[0;36m' COLOR_ERROR='\033[1;31m' -# clean cleans writer-001's cluster data and the `go run` process. +# clean cleans primary-001's cluster data and the `go run` process. function clean() { # basic cleaning routine @@ -65,8 +65,8 @@ function clean() # clean the cluster resource first. clean -# start writer01 for testing. -bash $WRITER01DIR/start.sh +# start primary01 for testing. +bash $PRIMARY01DIR/start.sh try_time=0 # wait Easegress to be ready, it will retry three times. # reference: https://unix.stackexchange.com/questions/5277/how-do-i-tell-a-script-to-wait-for-a-process-to-start-accepting-requests-on-a-po @@ -80,7 +80,7 @@ do fi done -# check the writer01 running status +# check the primary01 running status pid=`ps -eo pid,args | grep "$server" | grep -v grep | awk '{print $1}'` if [ "$pid" = "" ]; then echo -e "\n${COLOR_ERROR}start test server $server failed${COLOR_NONE}" @@ -101,7 +101,7 @@ https: false rules: - paths: - pathPrefix: /pipeline - backend: pipeline-demo' | $WRITER01DIR/egctl.sh object create + backend: pipeline-demo' | $PRIMARY01DIR/egctl.sh object create # create Pipeline echo ' @@ -116,7 +116,7 @@ filters: servers: - url: http://127.0.0.1:9095 loadBalance: - policy: roundRobin' | $WRITER01DIR/egctl.sh object create + policy: roundRobin' | $PRIMARY01DIR/egctl.sh object create while ! nc -z localhost $httpsvr_port /dev/null -exit 0 \ No newline at end of file +exit 0 diff --git a/example/README.md b/example/README.md index 7bca169947..421ca737b9 100644 --- a/example/README.md +++ b/example/README.md @@ -4,11 +4,11 @@ Port Layout: | Member | Cluster Client Port | Cluster Peer Port | API Port | | :--------: | :-----------------: | :---------------: | :------: | -| writer-001 | 12379 | 12380 | 12381 | -| writer-002 | 22379 | 22380 | 22381 | -| writer-003 | 32379 | 22380 | 32381 | -| reader-004 | - | - | 42381 | -| reader-005 | - | - | 52381 | +| primary-001 | 12379 | 12380 | 12381 | +| primary-002 | 22379 | 22380 | 22381 | +| primary-003 | 32379 | 22380 | 32381 | +| secondary-004 | - | - | 42381 | +| secondary-005 | - | - | 52381 | ## Start Easegress Cluster diff --git a/example/check_cluster_status.sh b/example/check_cluster_status.sh index 0b54ae0a24..da23c4a9fb 100755 --- a/example/check_cluster_status.sh +++ b/example/check_cluster_status.sh @@ -13,11 +13,11 @@ BLUE='\033[0;34m' NC='\033[0m' echo -e ${GREEN}"status:"${NC} -${SCRIPTPATH}/writer-001/status.sh +${SCRIPTPATH}/primary-001/status.sh echo -e ${GREEN}"members:"${NC} -${SCRIPTPATH}/writer-001/egctl.sh member list +${SCRIPTPATH}/primary-001/egctl.sh member list echo -e ${GREEN}"objects:"${NC} -${SCRIPTPATH}/writer-001/egctl.sh object list +${SCRIPTPATH}/primary-001/egctl.sh object list diff --git a/example/clean_cluster.sh b/example/clean_cluster.sh index 0bd17f03a7..53371c7b9d 100755 --- a/example/clean_cluster.sh +++ b/example/clean_cluster.sh @@ -5,7 +5,7 @@ SCRIPTPATH=`pwd -P` popd > /dev/null SCRIPTFILE=`basename $0` -for MEMBER_PATH in writer-00{1,2,3} reader-00{4,5} +for MEMBER_PATH in primary-00{1,2,3} secondary-00{4,5} do ${SCRIPTPATH}/${MEMBER_PATH}/stop.sh -f rm -fr ${SCRIPTPATH}/${MEMBER_PATH}/{nohup.out,log,member,data,data_bak,easegress.pid,running_objects.yaml} diff --git a/example/create_objects.sh b/example/create_objects.sh index 7a6fd38cca..cec9b7387d 100755 --- a/example/create_objects.sh +++ b/example/create_objects.sh @@ -9,5 +9,5 @@ CONFIG_PATH=${SCRIPTPATH}/config for CONFIG_FILE in ${CONFIG_PATH}/*.yaml do echo "create object: ${CONFIG_FILE}" - $SCRIPTPATH/writer-001/egctl.sh object create -f ${CONFIG_FILE} + $SCRIPTPATH/primary-001/egctl.sh object create -f ${CONFIG_FILE} done diff --git a/example/delete_objects.sh b/example/delete_objects.sh index c1abe03083..71841fd35f 100755 --- a/example/delete_objects.sh +++ b/example/delete_objects.sh @@ -13,5 +13,5 @@ for CONFIG_FILE in ${CONFIG_PATH}/*.yaml do OBJECT_NAME=`echo ${CONFIG_FILE} | sed -e "s%^${CONFIG_PATH}/%%" -e "s%\.yaml$%%"` echo "delete object: ${OBJECT_NAME}" - $SCRIPTPATH/writer-001/egctl.sh object delete ${OBJECT_NAME} + $SCRIPTPATH/primary-001/egctl.sh object delete ${OBJECT_NAME} done diff --git a/example/primary-001/conf/config.yaml b/example/primary-001/conf/config.yaml new file mode 100644 index 0000000000..24b5e448e2 --- /dev/null +++ b/example/primary-001/conf/config.yaml @@ -0,0 +1,19 @@ +name: primary-001 +cluster-name: cluster-test +cluster-role: primary +cluster: + listen-client-urls: [http://127.0.0.1:12379] + listen-peer-urls: [http://127.0.0.1:12380] + advertise-client-urls: [http://127.0.0.1:12379] + initial-advertise-peer-urls: [http://127.0.0.1:12380] + initial-cluster: + primary-001: http://127.0.0.1:12380 + primary-002: http://127.0.0.1:22380 + primary-003: http://127.0.0.1:32380 +api-addr: 127.0.0.1:12381 +data-dir: ./data +wal-dir: "" +cpu-profile-file: +memory-profile-file: +log-dir: ./log +debug: false diff --git a/example/reader-004/egctl.sh b/example/primary-001/egctl.sh similarity index 100% rename from example/reader-004/egctl.sh rename to example/primary-001/egctl.sh diff --git a/example/reader-004/start.sh b/example/primary-001/start.sh similarity index 100% rename from example/reader-004/start.sh rename to example/primary-001/start.sh diff --git a/example/reader-004/status.sh b/example/primary-001/status.sh similarity index 100% rename from example/reader-004/status.sh rename to example/primary-001/status.sh diff --git a/example/reader-004/stop.sh b/example/primary-001/stop.sh similarity index 100% rename from example/reader-004/stop.sh rename to example/primary-001/stop.sh diff --git a/example/primary-002/conf/config.yaml b/example/primary-002/conf/config.yaml new file mode 100644 index 0000000000..447c72af1e --- /dev/null +++ b/example/primary-002/conf/config.yaml @@ -0,0 +1,20 @@ +name: primary-002 +cluster-name: cluster-test +cluster-role: primary +cluster: + listen-client-urls: [http://127.0.0.1:22379] + listen-peer-urls: [http://127.0.0.1:22380] + advertise-client-urls: [http://127.0.0.1:22379] + initial-advertise-peer-urls: [http://127.0.0.1:22380] + initial-cluster: + primary-001: http://127.0.0.1:12380 + primary-002: http://127.0.0.1:22380 + primary-003: http://127.0.0.1:32380 +api-addr: 127.0.0.1:22381 +data-dir: ./data +wal-dir: "" +cpu-profile-file: +memory-profile-file: +log-dir: ./log +member-dir: ./member +debug: false diff --git a/example/reader-005/egctl.sh b/example/primary-002/egctl.sh similarity index 100% rename from example/reader-005/egctl.sh rename to example/primary-002/egctl.sh diff --git a/example/reader-005/start.sh b/example/primary-002/start.sh similarity index 100% rename from example/reader-005/start.sh rename to example/primary-002/start.sh diff --git a/example/reader-005/status.sh b/example/primary-002/status.sh similarity index 100% rename from example/reader-005/status.sh rename to example/primary-002/status.sh diff --git a/example/reader-005/stop.sh b/example/primary-002/stop.sh similarity index 100% rename from example/reader-005/stop.sh rename to example/primary-002/stop.sh diff --git a/example/primary-003/conf/config.yaml b/example/primary-003/conf/config.yaml new file mode 100644 index 0000000000..9abfb69d88 --- /dev/null +++ b/example/primary-003/conf/config.yaml @@ -0,0 +1,20 @@ +name: primary-003 +cluster-name: cluster-test +cluster-role: primary +cluster: + listen-client-urls: [http://127.0.0.1:32379] + listen-peer-urls: [http://127.0.0.1:32380] + advertise-client-urls: [http://127.0.0.1:32379] + initial-advertise-peer-urls: [http://127.0.0.1:32380] + initial-cluster: + primary-001: http://127.0.0.1:12380 + primary-002: http://127.0.0.1:22380 + primary-003: http://127.0.0.1:32380 +api-addr: 127.0.0.1:32381 +data-dir: ./data +wal-dir: "" +cpu-profile-file: +memory-profile-file: +log-dir: ./log +member-dir: ./member +debug: false diff --git a/example/writer-001/egctl.sh b/example/primary-003/egctl.sh similarity index 100% rename from example/writer-001/egctl.sh rename to example/primary-003/egctl.sh diff --git a/example/writer-001/start.sh b/example/primary-003/start.sh similarity index 100% rename from example/writer-001/start.sh rename to example/primary-003/start.sh diff --git a/example/writer-001/status.sh b/example/primary-003/status.sh similarity index 100% rename from example/writer-001/status.sh rename to example/primary-003/status.sh diff --git a/example/writer-001/stop.sh b/example/primary-003/stop.sh similarity index 100% rename from example/writer-001/stop.sh rename to example/primary-003/stop.sh diff --git a/example/reader-004/README.md b/example/reader-004/README.md deleted file mode 120000 index 2eca3ce434..0000000000 --- a/example/reader-004/README.md +++ /dev/null @@ -1 +0,0 @@ -../../sbin/README.md \ No newline at end of file diff --git a/example/reader-004/bin b/example/reader-004/bin deleted file mode 120000 index db397da27f..0000000000 --- a/example/reader-004/bin +++ /dev/null @@ -1 +0,0 @@ -../../bin \ No newline at end of file diff --git a/example/reader-004/conf/config.yaml b/example/reader-004/conf/config.yaml deleted file mode 100644 index 22580be3c5..0000000000 --- a/example/reader-004/conf/config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: reader-004 -cluster-name: cluster-test -cluster-role: reader -cluster-listen-client-urls: -cluster-listen-peer-urls: -cluster-advertise-client-urls: -cluster-initial-advertise-peer-urls: -cluster-join-urls: [http://127.0.0.1:12380, http://127.0.0.1:22380, http://127.0.0.1:32380] -api-addr: 127.0.0.1:42381 -data-dir: ./data -wal-dir: "" -cpu-profile-file: -memory-profile-file: -log-dir: ./log -member-dir: ./member -debug: false diff --git a/example/reader-005/README.md b/example/reader-005/README.md deleted file mode 120000 index 2eca3ce434..0000000000 --- a/example/reader-005/README.md +++ /dev/null @@ -1 +0,0 @@ -../../sbin/README.md \ No newline at end of file diff --git a/example/reader-005/bin b/example/reader-005/bin deleted file mode 120000 index db397da27f..0000000000 --- a/example/reader-005/bin +++ /dev/null @@ -1 +0,0 @@ -../../bin \ No newline at end of file diff --git a/example/reader-005/conf/config.yaml b/example/reader-005/conf/config.yaml deleted file mode 100644 index 0899ff8c5c..0000000000 --- a/example/reader-005/conf/config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: reader-005 -cluster-name: cluster-test -cluster-role: reader -cluster-listen-client-urls: -cluster-listen-peer-urls: -cluster-advertise-client-urls: -cluster-initial-advertise-peer-urls: -cluster-join-urls: [http://127.0.0.1:12380, http://127.0.0.1:22380, http://127.0.0.1:32380] -api-addr: 127.0.0.1:52381 -data-dir: ./data -wal-dir: "" -cpu-profile-file: -memory-profile-file: -log-dir: ./log -member-dir: ./member -debug: false diff --git a/example/sbin/stop.sh b/example/sbin/stop.sh index 86d472c698..0ace805757 100755 --- a/example/sbin/stop.sh +++ b/example/sbin/stop.sh @@ -30,8 +30,8 @@ fi RED='\033[0;31m' NC='\033[0m' # No Color -total_etcd=`./status.sh | grep writer | wc -l` -online=`./status.sh | grep writer | grep online | wc -l` +total_etcd=`./status.sh | grep primary | wc -l` +online=`./status.sh | grep primary | grep online | wc -l` if [ $online -le $((total_etcd / 2 + 1)) ] then diff --git a/example/secondary-004/conf/config.yaml b/example/secondary-004/conf/config.yaml new file mode 100644 index 0000000000..3eddec01b6 --- /dev/null +++ b/example/secondary-004/conf/config.yaml @@ -0,0 +1,14 @@ +name: secondary-004 +cluster-name: cluster-test +cluster-role: secondary +cluster: + primary-listen-peer-urls: + - http://127.0.0.1:12380 +api-addr: 127.0.0.1:42381 +data-dir: ./data +wal-dir: "" +cpu-profile-file: +memory-profile-file: +log-dir: ./log +member-dir: ./member +debug: false diff --git a/example/writer-002/egctl.sh b/example/secondary-004/egctl.sh similarity index 100% rename from example/writer-002/egctl.sh rename to example/secondary-004/egctl.sh diff --git a/example/writer-002/start.sh b/example/secondary-004/start.sh similarity index 100% rename from example/writer-002/start.sh rename to example/secondary-004/start.sh diff --git a/example/writer-002/status.sh b/example/secondary-004/status.sh similarity index 100% rename from example/writer-002/status.sh rename to example/secondary-004/status.sh diff --git a/example/writer-002/stop.sh b/example/secondary-004/stop.sh similarity index 100% rename from example/writer-002/stop.sh rename to example/secondary-004/stop.sh diff --git a/example/secondary-005/conf/config.yaml b/example/secondary-005/conf/config.yaml new file mode 100644 index 0000000000..a57893b3ed --- /dev/null +++ b/example/secondary-005/conf/config.yaml @@ -0,0 +1,14 @@ +name: secondary-005 +cluster-name: cluster-test +cluster-role: secondary +cluster: + primary-listen-peer-urls: + - http://127.0.0.1:12380 +api-addr: 127.0.0.1:52381 +data-dir: ./data +wal-dir: "" +cpu-profile-file: +memory-profile-file: +log-dir: ./log +member-dir: ./member +debug: false diff --git a/example/writer-003/egctl.sh b/example/secondary-005/egctl.sh similarity index 100% rename from example/writer-003/egctl.sh rename to example/secondary-005/egctl.sh diff --git a/example/writer-003/start.sh b/example/secondary-005/start.sh similarity index 100% rename from example/writer-003/start.sh rename to example/secondary-005/start.sh diff --git a/example/writer-003/status.sh b/example/secondary-005/status.sh similarity index 100% rename from example/writer-003/status.sh rename to example/secondary-005/status.sh diff --git a/example/writer-003/stop.sh b/example/secondary-005/stop.sh similarity index 100% rename from example/writer-003/stop.sh rename to example/secondary-005/stop.sh diff --git a/example/start_cluster.sh b/example/start_cluster.sh index 7855de5d99..f46089ce74 100755 --- a/example/start_cluster.sh +++ b/example/start_cluster.sh @@ -8,7 +8,7 @@ SCRIPTFILE=`basename $0` REPOPATH=${SCRIPTPATH}/.. -for MEMBER_PATH in writer-00{1,2,3} reader-00{4,5} +for MEMBER_PATH in primary-00{1,2,3} secondary-00{4,5} do echo "start ${MEMBER_PATH}" ${SCRIPTPATH}/${MEMBER_PATH}/start.sh diff --git a/example/stop_cluster.sh b/example/stop_cluster.sh index 99730257a8..f9e3afb175 100755 --- a/example/stop_cluster.sh +++ b/example/stop_cluster.sh @@ -5,7 +5,7 @@ SCRIPTPATH=`pwd -P` popd > /dev/null SCRIPTFILE=`basename $0` -for MEMBER_PATH in writer-00{1,2,3} reader-00{4,5} +for MEMBER_PATH in primary-00{1,2,3} secondary-00{4,5} do echo "stop ${MEMBER_PATH}" ${SCRIPTPATH}/${MEMBER_PATH}/stop.sh -f diff --git a/example/update_objects.sh b/example/update_objects.sh index 0328542e39..c0c70f08b5 100755 --- a/example/update_objects.sh +++ b/example/update_objects.sh @@ -12,5 +12,5 @@ CONFIG_PATH=${SCRIPTPATH}/config for CONFIG_FILE in ${CONFIG_PATH}/*.yaml do echo "update object: ${CONFIG_FILE}" - $SCRIPTPATH/writer-001/egctl.sh object update -f ${CONFIG_FILE} + $SCRIPTPATH/primary-001/egctl.sh object update -f ${CONFIG_FILE} done diff --git a/example/writer-001/bin b/example/writer-001/bin deleted file mode 120000 index db397da27f..0000000000 --- a/example/writer-001/bin +++ /dev/null @@ -1 +0,0 @@ -../../bin \ No newline at end of file diff --git a/example/writer-001/conf/config.yaml b/example/writer-001/conf/config.yaml deleted file mode 100644 index 36d5269ad7..0000000000 --- a/example/writer-001/conf/config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: writer-001 -cluster-name: cluster-test -cluster-role: primary -cluster-listen-client-urls: [http://127.0.0.1:12379] -cluster-listen-peer-urls: [http://127.0.0.1:12380] -cluster-advertise-client-urls: [http://127.0.0.1:12379] -cluster-initial-advertise-peer-urls: [http://127.0.0.1:12380] -cluster-join-urls: [http://127.0.0.1:12380, http://127.0.0.1:22380, http://127.0.0.1:32380] -api-addr: 127.0.0.1:12381 -data-dir: ./data -wal-dir: "" -cpu-profile-file: -memory-profile-file: -log-dir: ./log -member-dir: ./member -debug: false diff --git a/example/writer-002/README.md b/example/writer-002/README.md deleted file mode 120000 index 2eca3ce434..0000000000 --- a/example/writer-002/README.md +++ /dev/null @@ -1 +0,0 @@ -../../sbin/README.md \ No newline at end of file diff --git a/example/writer-002/bin b/example/writer-002/bin deleted file mode 120000 index db397da27f..0000000000 --- a/example/writer-002/bin +++ /dev/null @@ -1 +0,0 @@ -../../bin \ No newline at end of file diff --git a/example/writer-002/conf/config.yaml b/example/writer-002/conf/config.yaml deleted file mode 100644 index 2bd78ed948..0000000000 --- a/example/writer-002/conf/config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: writer-002 -cluster-name: cluster-test -cluster-role: primary -cluster-listen-client-urls: [http://127.0.0.1:22379] -cluster-listen-peer-urls: [http://127.0.0.1:22380] -cluster-advertise-client-urls: [http://127.0.0.1:22379] -cluster-initial-advertise-peer-urls: [http://127.0.0.1:22380] -cluster-join-urls: [http://127.0.0.1:12380, http://127.0.0.1:22380, http://127.0.0.1:32380] -api-addr: 127.0.0.1:22381 -data-dir: ./data -wal-dir: "" -cpu-profile-file: -memory-profile-file: -log-dir: ./log -member-dir: ./member -debug: false diff --git a/example/writer-003/README.md b/example/writer-003/README.md deleted file mode 120000 index 2eca3ce434..0000000000 --- a/example/writer-003/README.md +++ /dev/null @@ -1 +0,0 @@ -../../sbin/README.md \ No newline at end of file diff --git a/example/writer-003/bin b/example/writer-003/bin deleted file mode 120000 index db397da27f..0000000000 --- a/example/writer-003/bin +++ /dev/null @@ -1 +0,0 @@ -../../bin \ No newline at end of file diff --git a/example/writer-003/conf/config.yaml b/example/writer-003/conf/config.yaml deleted file mode 100644 index 65de519b40..0000000000 --- a/example/writer-003/conf/config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: writer-003 -cluster-name: cluster-test -cluster-role: primary -cluster-listen-client-urls: [http://127.0.0.1:32379] -cluster-listen-peer-urls: [http://127.0.0.1:32380] -cluster-advertise-client-urls: [http://127.0.0.1:32379] -cluster-initial-advertise-peer-urls: [http://127.0.0.1:32380] -cluster-join-urls: [http://127.0.0.1:12380, http://127.0.0.1:22380, http://127.0.0.1:32380] -api-addr: 127.0.0.1:32381 -data-dir: ./data -wal-dir: "" -cpu-profile-file: -memory-profile-file: -log-dir: ./log -member-dir: ./member -debug: false From d60cab7d95870c7d4424883dbe5ae22b14450801 Mon Sep 17 00:00:00 2001 From: Samu Tamminen Date: Fri, 26 Nov 2021 16:03:45 +0800 Subject: [PATCH 15/17] revert config update for example/primary*/config as httpserver_test use legacy cluster startup routine. Remove update fnctions from pkg/option/option --- example/primary-001/conf/config.yaml | 13 ++++------- example/primary-002/conf/config.yaml | 14 ++++-------- example/primary-003/conf/config.yaml | 14 ++++-------- pkg/cluster/cluster.go | 31 +++++-------------------- pkg/option/option.go | 34 ++++++++++------------------ 5 files changed, 33 insertions(+), 73 deletions(-) diff --git a/example/primary-001/conf/config.yaml b/example/primary-001/conf/config.yaml index 24b5e448e2..82ce2d1ac9 100644 --- a/example/primary-001/conf/config.yaml +++ b/example/primary-001/conf/config.yaml @@ -2,14 +2,11 @@ name: primary-001 cluster-name: cluster-test cluster-role: primary cluster: - listen-client-urls: [http://127.0.0.1:12379] - listen-peer-urls: [http://127.0.0.1:12380] - advertise-client-urls: [http://127.0.0.1:12379] - initial-advertise-peer-urls: [http://127.0.0.1:12380] - initial-cluster: - primary-001: http://127.0.0.1:12380 - primary-002: http://127.0.0.1:22380 - primary-003: http://127.0.0.1:32380 +cluster-listen-client-urls: [http://127.0.0.1:12379] +cluster-listen-peer-urls: [http://127.0.0.1:12380] +cluster-advertise-client-urls: [http://127.0.0.1:12379] +cluster-initial-advertise-peer-urls: [http://127.0.0.1:12380] +cluster-join-urls: [http://127.0.0.1:12380,http://127.0.0.1:22380,http://127.0.0.1:32380] api-addr: 127.0.0.1:12381 data-dir: ./data wal-dir: "" diff --git a/example/primary-002/conf/config.yaml b/example/primary-002/conf/config.yaml index 447c72af1e..2d947173a4 100644 --- a/example/primary-002/conf/config.yaml +++ b/example/primary-002/conf/config.yaml @@ -1,15 +1,11 @@ name: primary-002 cluster-name: cluster-test cluster-role: primary -cluster: - listen-client-urls: [http://127.0.0.1:22379] - listen-peer-urls: [http://127.0.0.1:22380] - advertise-client-urls: [http://127.0.0.1:22379] - initial-advertise-peer-urls: [http://127.0.0.1:22380] - initial-cluster: - primary-001: http://127.0.0.1:12380 - primary-002: http://127.0.0.1:22380 - primary-003: http://127.0.0.1:32380 +cluster-listen-client-urls: [http://127.0.0.1:22379] +cluster-listen-peer-urls: [http://127.0.0.1:22380] +cluster-advertise-client-urls: [http://127.0.0.1:22379] +cluster-initial-advertise-peer-urls: [http://127.0.0.1:22380] +cluster-join-urls: [http://127.0.0.1:12380,http://127.0.0.1:22380,http://127.0.0.1:32380] api-addr: 127.0.0.1:22381 data-dir: ./data wal-dir: "" diff --git a/example/primary-003/conf/config.yaml b/example/primary-003/conf/config.yaml index 9abfb69d88..9948c2a210 100644 --- a/example/primary-003/conf/config.yaml +++ b/example/primary-003/conf/config.yaml @@ -1,15 +1,11 @@ name: primary-003 cluster-name: cluster-test cluster-role: primary -cluster: - listen-client-urls: [http://127.0.0.1:32379] - listen-peer-urls: [http://127.0.0.1:32380] - advertise-client-urls: [http://127.0.0.1:32379] - initial-advertise-peer-urls: [http://127.0.0.1:32380] - initial-cluster: - primary-001: http://127.0.0.1:12380 - primary-002: http://127.0.0.1:22380 - primary-003: http://127.0.0.1:32380 +cluster-listen-client-urls: [http://127.0.0.1:32379] +cluster-listen-peer-urls: [http://127.0.0.1:32380] +cluster-advertise-client-urls: [http://127.0.0.1:32379] +cluster-initial-advertise-peer-urls: [http://127.0.0.1:32380] +cluster-join-urls: [http://127.0.0.1:12380,http://127.0.0.1:22380,http://127.0.0.1:32380] api-addr: 127.0.0.1:32381 data-dir: ./data wal-dir: "" diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index ffb12e68a6..7abc6a80e9 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -26,7 +26,6 @@ import ( "sync" "time" - pb "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" "go.etcd.io/etcd/server/v3/embed" @@ -750,9 +749,11 @@ func (c *cluster) defrag() { defragInterval = defragFailedInterval logger.Errorf("defrag failed: get client failed: %v", err) } - defragmentURL := c.opt.ClusterAdvertiseClientURLs[0] - if c.opt.UseInitialCluster() { - defragmentURL = c.opt.Cluster.AdvertiseClientURLs[0] + + defragmentURL, err := c.opt.GetFirstAdvertiseClientURL() + if err != nil { + logger.Errorf("defrag failed: %v", err) + return } // NOTICE: It needs longer time than normal ones. _, err = client.Defragment(c.longRequestContext(), defragmentURL) @@ -814,32 +815,12 @@ func (c *cluster) updateMembers() error { return err } - if c.members == nil { - c.UpdatePeerURLs(resp.Members) - } else { + if c.members != nil { c.members.updateClusterMembers(resp.Members) } return nil } -// UpdatePeerURLs updates peerURLs according the cluster role. -func (c *cluster) UpdatePeerURLs(pbMembers []*pb.Member) { - primaryMembers := pbMembersToMembersSlice(pbMembers) - if c.opt.ClusterRole == "secondary" { - newPeerURLs := make([]string, 0) - for _, member := range primaryMembers { - newPeerURLs = append(newPeerURLs, member.PeerURL) - } - c.opt.SetClusterPrimaryListenPeerURLs(newPeerURLs) - } else { - newInitialCluster := make(map[string]string) - for _, member := range primaryMembers { - newInitialCluster[member.Name] = member.PeerURL - } - c.opt.SetInitialCluster(newInitialCluster) - } -} - func (c *cluster) PurgeMember(memberName string) error { client, err := c.getClient() if err != nil { diff --git a/pkg/option/option.go b/pkg/option/option.go index fd2b4c3a5e..89a8e9f184 100644 --- a/pkg/option/option.go +++ b/pkg/option/option.go @@ -24,7 +24,6 @@ import ( "os" "path/filepath" "strings" - "sync" "time" "github.com/mitchellh/mapstructure" @@ -101,9 +100,6 @@ type Options struct { AbsWALDir string `yaml:"-"` AbsLogDir string `yaml:"-"` AbsMemberDir string `yaml:"-"` - - // For Options in-memory updates. Updates are not persisted. - mutex sync.Mutex } // addClusterVars introduces cluster arguments. @@ -139,7 +135,6 @@ func New() *Options { opt := &Options{ flags: pflag.NewFlagSet(os.Args[0], pflag.ContinueOnError), viper: viper.New(), - mutex: sync.Mutex{}, } opt.flags.BoolVarP(&opt.ShowVersion, "version", "v", false, "Print the version and exit.") @@ -470,9 +465,6 @@ func (opt *Options) InitialClusterToString() string { // GetPeerURLs returns URLs listed in cluster.initial-cluster for primary (a.k.a writer) and // for secondary (a.k.a reader) the ones listed in cluster.primary-listen-peer-url. func (opt *Options) GetPeerURLs() []string { - opt.mutex.Lock() - defer opt.mutex.Unlock() - if opt.ClusterRole == "secondary" { return opt.Cluster.PrimaryListenPeerURLs } @@ -483,20 +475,18 @@ func (opt *Options) GetPeerURLs() []string { return peerURLs } -// SetClusterPrimaryListenPeerURLs updates secondary role's peerURLs. -func (opt *Options) SetClusterPrimaryListenPeerURLs(peerURLs []string) { - opt.mutex.Lock() - defer opt.mutex.Unlock() - - opt.Cluster.PrimaryListenPeerURLs = peerURLs -} - -// SetInitialCluster updates primary role's initial cluster. -func (opt *Options) SetInitialCluster(peerURLs map[string]string) { - opt.mutex.Lock() - defer opt.mutex.Unlock() - - opt.Cluster.InitialCluster = peerURLs +// GetFirstAdvertiseClientURL returns the first advertised client url. +func (opt *Options) GetFirstAdvertiseClientURL() (string, error) { + if opt.UseInitialCluster() { + if len(opt.Cluster.AdvertiseClientURLs) == 0 { + return "", fmt.Errorf("cluster.advertise-client-URLs is empty") + } + return opt.Cluster.AdvertiseClientURLs[0], nil + } + if len(opt.ClusterAdvertiseClientURLs) == 0 { + return "", fmt.Errorf("cluster-advertise-client-URLs is empty") + } + return opt.ClusterAdvertiseClientURLs[0], nil } func generateMemberName(apiAddr string) (string, error) { From e5dbb8d1ad5677731f193388c939274f436e7087 Mon Sep 17 00:00:00 2001 From: Samu Tamminen Date: Fri, 26 Nov 2021 16:49:00 +0800 Subject: [PATCH 16/17] update readme --- README.md | 63 ++++++++++++++++++------------------------------- README.zh-CN.md | 62 ++++++++++++++++++------------------------------ 2 files changed, 46 insertions(+), 79 deletions(-) diff --git a/README.md b/README.md index 02f5a1467a..6eb428e478 100644 --- a/README.md +++ b/README.md @@ -146,18 +146,22 @@ Then we can add the binary directory to the `PATH` and execute the server: ```bash $ export PATH=${PATH}:$(pwd)/bin/ $ easegress-server -2021-05-17T16:45:38.185+08:00 INFO cluster/config.go:84 etcd config: init-cluster:eg-default-name=http://localhost:2380 cluster-state:new force-new-cluster:false -2021-05-17T16:45:38.185+08:00 INFO cluster/cluster.go:379 client is ready -2021-05-17T16:45:39.189+08:00 INFO cluster/cluster.go:590 server is ready -2021-05-17T16:45:39.21+08:00 INFO cluster/cluster.go:451 lease is ready -2021-05-17T16:45:39.231+08:00 INFO cluster/cluster.go:187 cluster is ready -2021-05-17T16:45:39.253+08:00 INFO supervisor/supervisor.go:180 create system controller StatusSyncController -2021-05-17T16:45:39.253+08:00 INFO cluster/cluster.go:496 session is ready -2021-05-17T16:45:39.253+08:00 INFO api/api.go:96 api server running in localhost:2381 -2021-05-17T16:45:44.235+08:00 INFO cluster/member.go:210 self ID changed from 0 to 689e371e88f78b6a -2021-05-17T16:45:44.236+08:00 INFO cluster/member.go:137 store clusterMembers: eg-default-name(689e371e88f78b6a)=http://localhost:2380 -2021-05-17T16:45:44.236+08:00 INFO cluster/member.go:138 store knownMembers : eg-default-name(689e371e88f78b6a)=http://localhost:2380 - +2021-11-26T16:10:08.419+08:00 INFO cluster/config.go:172 etcd config: init-cluster:eg-default-name=http://localhost:2380 cluster-state:new force-new-cluster:false +2021-11-26T16:10:09.515+08:00 INFO cluster/cluster.go:400 client connect with endpoints: [http://localhost:2380] +2021-11-26T16:10:09.516+08:00 INFO cluster/cluster.go:413 client is ready +2021-11-26T16:10:09.608+08:00 INFO cluster/cluster.go:692 server is ready +2021-11-26T16:10:09.649+08:00 INFO cluster/cluster.go:546 lease is ready (grant new one: b6a7d5b4b68ff07) +2021-11-26T16:10:09.649+08:00 INFO cluster/cluster.go:219 cluster is ready +2021-11-26T16:10:09.669+08:00 INFO supervisor/supervisor.go:137 create TrafficController +2021-11-26T16:10:09.67+08:00 INFO supervisor/supervisor.go:137 create RawConfigTrafficController +2021-11-26T16:10:09.67+08:00 INFO supervisor/supervisor.go:137 create ServiceRegistry +2021-11-26T16:10:09.671+08:00 INFO supervisor/supervisor.go:137 create StatusSyncController +2021-11-26T16:10:09.671+08:00 INFO cluster/cluster.go:586 session is ready +2021-11-26T16:10:09.671+08:00 INFO api/api.go:73 register api group admin +2021-11-26T16:10:09.671+08:00 INFO api/server.go:78 api server running in localhost:2381 +2021-11-26T16:10:14.673+08:00 INFO cluster/member.go:223 self ID changed from 0 to 689e371e88f78b6a +2021-11-26T16:10:14.674+08:00 INFO cluster/member.go:157 store clusterMembers: eg-default-name(689e371e88f78b6a)=http://localhost:2380 +2021-11-26T16:10:14.674+08:00 INFO cluster/member.go:158 store knownMembers : eg-default-name(689e371e88f78b6a)=http://localhost:2380 ``` The default target of Makefile is to compile two binary into the directory `bin/`. `bin/easegress-server` is the server-side binary, `bin/egctl` is the client-side binary. We could add it to the `$PATH` for simplifying the following commands. @@ -165,36 +169,15 @@ The default target of Makefile is to compile two binary into the directory `bin/ We could run `easegress-server` without specifying any arguments, which launch itself by opening default ports 2379, 2380, 2381. Of course, we can change them in the config file or command arguments that are explained well in `easegress-server --help`. ```bash -$ egctl member list -- options: - name: eg-default-name - labels: {} - cluster-name: eg-cluster-default-name +$ egctl member list | grep "cluster-role" cluster-role: primary - cluster-request-timeout: 10s - cluster-listen-client-urls: - - http://127.0.0.1:2379 - cluster-listen-peer-urls: - - http://127.0.0.1:2380 - cluster-advertise-client-urls: - - http://127.0.0.1:2379 - cluster-initial-advertise-peer-urls: - - http://127.0.0.1:2380 - cluster-join-urls: [] +$ egctl member list | grep "api-addr" api-addr: localhost:2381 - debug: false - home-dir: ./ - data-dir: data - wal-dir: "" - log-dir: log - member-dir: member - cpu-profile-file: "" - memory-profile-file: "" - lastHeartbeatTime: "2021-05-05T15:43:27+08:00" - etcd: - id: a30c34bf7ec77546 - startTime: "2021-05-05T15:42:37+08:00" - state: Leader +$ egctl member list | grep "name" + name: eg-default-name + cluster-name: eg-cluster-default-name +$ egctl member list | grep "id" + id: 689e371e88f78b6a ``` After launched successfully, we could check the status of the one-node cluster. It shows the static options and dynamic status of heartbeat and etcd. diff --git a/README.zh-CN.md b/README.zh-CN.md index f4ccdce665..94fb779aff 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -146,17 +146,22 @@ make ```bash $ export PATH=${PATH}:$(pwd)/bin/ $ easegress-server -2021-05-17T16:45:38.185+08:00 INFO cluster/config.go:84 etcd config: init-cluster:eg-default-name=http://localhost:2380 cluster-state:new force-new-cluster:false -2021-05-17T16:45:38.185+08:00 INFO cluster/cluster.go:379 client is ready -2021-05-17T16:45:39.189+08:00 INFO cluster/cluster.go:590 server is ready -2021-05-17T16:45:39.21+08:00 INFO cluster/cluster.go:451 lease is ready -2021-05-17T16:45:39.231+08:00 INFO cluster/cluster.go:187 cluster is ready -2021-05-17T16:45:39.253+08:00 INFO supervisor/supervisor.go:180 create system controller StatusSyncController -2021-05-17T16:45:39.253+08:00 INFO cluster/cluster.go:496 session is ready -2021-05-17T16:45:39.253+08:00 INFO api/api.go:96 api server running in localhost:2381 -2021-05-17T16:45:44.235+08:00 INFO cluster/member.go:210 self ID changed from 0 to 689e371e88f78b6a -2021-05-17T16:45:44.236+08:00 INFO cluster/member.go:137 store clusterMembers: eg-default-name(689e371e88f78b6a)=http://localhost:2380 -2021-05-17T16:45:44.236+08:00 INFO cluster/member.go:138 store knownMembers : eg-default-name(689e371e88f78b6a)=http://localhost:2380 +2021-11-26T16:10:08.419+08:00 INFO cluster/config.go:172 etcd config: init-cluster:eg-default-name=http://localhost:2380 cluster-state:new force-new-cluster:false +2021-11-26T16:10:09.515+08:00 INFO cluster/cluster.go:400 client connect with endpoints: [http://localhost:2380] +2021-11-26T16:10:09.516+08:00 INFO cluster/cluster.go:413 client is ready +2021-11-26T16:10:09.608+08:00 INFO cluster/cluster.go:692 server is ready +2021-11-26T16:10:09.649+08:00 INFO cluster/cluster.go:546 lease is ready (grant new one: b6a7d5b4b68ff07) +2021-11-26T16:10:09.649+08:00 INFO cluster/cluster.go:219 cluster is ready +2021-11-26T16:10:09.669+08:00 INFO supervisor/supervisor.go:137 create TrafficController +2021-11-26T16:10:09.67+08:00 INFO supervisor/supervisor.go:137 create RawConfigTrafficController +2021-11-26T16:10:09.67+08:00 INFO supervisor/supervisor.go:137 create ServiceRegistry +2021-11-26T16:10:09.671+08:00 INFO supervisor/supervisor.go:137 create StatusSyncController +2021-11-26T16:10:09.671+08:00 INFO cluster/cluster.go:586 session is ready +2021-11-26T16:10:09.671+08:00 INFO api/api.go:73 register api group admin +2021-11-26T16:10:09.671+08:00 INFO api/server.go:78 api server running in localhost:2381 +2021-11-26T16:10:14.673+08:00 INFO cluster/member.go:223 self ID changed from 0 to 689e371e88f78b6a +2021-11-26T16:10:14.674+08:00 INFO cluster/member.go:157 store clusterMembers: eg-default-name(689e371e88f78b6a)=http://localhost:2380 +2021-11-26T16:10:14.674+08:00 INFO cluster/member.go:158 store knownMembers : eg-default-name(689e371e88f78b6a)=http://localhost:2380 ``` Makefile 默认会将两个二进制文件编译到 `bin/` 目录中。`bin/easegress-server` 是服务器端的二进制文件，`bin/egctl` 是客户端的二进制文件。我们可以把它添加到 `$PATH` 中，以便于执行后续命令。 @@ -164,36 +169,15 @@ Makefile 默认会将两个二进制文件编译到 `bin/` 目录中。`bin/ease 如果启动时不指定任何参数，`easegress-server` 会默认使用端口 2379、2380 和 2381。我们可以在配置文件中更改默认端口，或者在命令行启动时指定相关参数（参数具体释义可通过执行 `easegress-server --help` 命令获取）。 ```bash -$ egctl member list -- options: - name: eg-default-name - labels: {} - cluster-name: eg-cluster-default-name +$ egctl member list | grep "cluster-role" cluster-role: primary - cluster-request-timeout: 10s - cluster-listen-client-urls: - - http://127.0.0.1:2379 - cluster-listen-peer-urls: - - http://127.0.0.1:2380 - cluster-advertise-client-urls: - - http://127.0.0.1:2379 - cluster-initial-advertise-peer-urls: - - http://127.0.0.1:2380 - cluster-join-urls: [] +$ egctl member list | grep "api-addr" api-addr: localhost:2381 - debug: false - home-dir: ./ - data-dir: data - wal-dir: "" - log-dir: log - member-dir: member - cpu-profile-file: "" - memory-profile-file: "" - lastHeartbeatTime: "2021-05-05T15:43:27+08:00" - etcd: - id: a30c34bf7ec77546 - startTime: "2021-05-05T15:42:37+08:00" - state: Leader +$ egctl member list | grep "name" + name: eg-default-name + cluster-name: eg-cluster-default-name +$ egctl member list | grep "id" + id: 689e371e88f78b6a ``` 成功启动后，我们可以用上述命令检查单节点集群的状态，它展示示了系统的静态选项，以及心跳和etcd的动态状态。 From 9400277eccfbcaf9b4e4a925a9b77d36e1f6938a Mon Sep 17 00:00:00 2001 From: Samu Tamminen Date: Fri, 26 Nov 2021 17:37:49 +0800 Subject: [PATCH 17/17] add missing symbolic links to example --- example/primary-001/bin | 1 + example/primary-002/bin | 1 + example/primary-003/bin | 1 + 3 files changed, 3 insertions(+) create mode 120000 example/primary-001/bin create mode 120000 example/primary-002/bin create mode 120000 example/primary-003/bin diff --git a/example/primary-001/bin b/example/primary-001/bin new file mode 120000 index 0000000000..db397da27f --- /dev/null +++ b/example/primary-001/bin @@ -0,0 +1 @@ +../../bin \ No newline at end of file diff --git a/example/primary-002/bin b/example/primary-002/bin new file mode 120000 index 0000000000..db397da27f --- /dev/null +++ b/example/primary-002/bin @@ -0,0 +1 @@ +../../bin \ No newline at end of file diff --git a/example/primary-003/bin b/example/primary-003/bin new file mode 120000 index 0000000000..db397da27f --- /dev/null +++ b/example/primary-003/bin @@ -0,0 +1 @@ +../../bin \ No newline at end of file