Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

roachprod: introduce --skip-init to roachprod start #53113

Merged
merged 1 commit into from
Aug 20, 2020
Merged

roachprod: introduce --skip-init to roachprod start #53113

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 41 additions & 0 deletions pkg/cmd/roachprod/install/cluster_synced.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import (
clog "github.com/cockroachdb/cockroach/pkg/util/log"
"github.com/cockroachdb/cockroach/pkg/util/syncutil"
"github.com/cockroachdb/cockroach/pkg/util/timeutil"
"github.com/cockroachdb/cockroach/pkg/util/version"
"github.com/cockroachdb/errors"
crdberrors "github.com/cockroachdb/errors"
"golang.org/x/sync/errgroup"
Expand Down Expand Up @@ -1736,3 +1737,43 @@ func (c *SyncedCluster) Parallel(
func (c *SyncedCluster) escapedTag() string {
return strings.Replace(c.Tag, "/", "\\/", -1)
}

// Init initializes the cluster. It does it through node 1 (as per ServerNodes)
// to maintain parity with auto-init behavior of `roachprod start` (when
// --skip-init) is not specified. The implementation should be kept in
// sync with Cockroach.Start.
func (c *SyncedCluster) Init() {
r := c.Impl.(Cockroach)
h := &crdbInstallHelper{c: c, r: r}

// See (Cockroach).Start. We reserve a few special operations for the first
// node, so we strive to maintain the same here for interoperability.
const firstNodeIdx = 0

vers, err := getCockroachVersion(c, c.ServerNodes()[firstNodeIdx])
if err != nil {
log.Fatalf("unable to retrieve cockroach version: %v", err)
}

if !vers.AtLeast(version.MustParse("v20.1.0")) {
log.Fatal("`roachprod init` only supported for v20.1 and beyond")
}

fmt.Printf("%s: initializing cluster\n", h.c.Name)
initOut, err := h.initializeCluster(firstNodeIdx)
if err != nil {
log.Fatalf("unable to initialize cluster: %v", err)
}
if initOut != "" {
fmt.Println(initOut)
}

fmt.Printf("%s: setting cluster settings\n", h.c.Name)
clusterSettingsOut, err := h.setClusterSettings(firstNodeIdx)
if err != nil {
log.Fatalf("unable to set cluster settings: %v", err)
}
if clusterSettingsOut != "" {
fmt.Println(clusterSettingsOut)
}
}
50 changes: 30 additions & 20 deletions pkg/cmd/roachprod/install/cockroach.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
var StartOpts struct {
Encrypt bool
Sequential bool
SkipInit bool
}

// Cockroach TODO(peter): document
Expand Down Expand Up @@ -119,7 +120,7 @@ func argExists(args []string, target string) int {
// `start-single-node` (this was written to provide a short hand to start a
// single node cluster with a replication factor of one).
func (r Cockroach) Start(c *SyncedCluster, extraArgs []string) {
h := &crdbStartHelper{c: c, r: r}
h := &crdbInstallHelper{c: c, r: r}
h.distributeCerts()

nodes := c.ServerNodes()
Expand Down Expand Up @@ -150,11 +151,17 @@ func (r Cockroach) Start(c *SyncedCluster, extraArgs []string) {
// NB: The code blocks below are not parallelized, so it's safe for us
// to use fmt.Printf style logging.

// 1. We don't init when invoking with `start-single-node`.
// 2. For nodes running <20.1, the --join flags are constructed in a manner
// such that the first node doesn't have any (see `generateStartArgs`),
// which prompts CRDB to auto-initialize. For nodes running >=20.1, we
// need to explicitly initialize.
// 1. We don't init invoked using `--skip-init`.
// 2. We don't init when invoking with `start-single-node`.
// 3. For nodes running <20.1, the --join flags are constructed in a
// manner such that the first node doesn't have any (see
// `generateStartArgs`),which prompts CRDB to auto-initialize. For
// nodes running >=20.1, we need to explicitly initialize.

if StartOpts.SkipInit {
return nil, nil
}

shouldInit := !h.useStartSingleNode(vers) && vers.AtLeast(version.MustParse("v20.1.0"))
if shouldInit {
fmt.Printf("%s: initializing cluster\n", h.c.Name)
Expand Down Expand Up @@ -188,6 +195,9 @@ func (r Cockroach) Start(c *SyncedCluster, extraArgs []string) {
}
}

// We're sure to set cluster settings after having initialized the
// cluster.

fmt.Printf("%s: setting cluster settings\n", h.c.Name)
clusterSettingsOut, err := h.setClusterSettings(nodeIdx)
if err != nil {
Expand Down Expand Up @@ -316,12 +326,12 @@ func (r Cockroach) SQL(c *SyncedCluster, args []string) error {
return nil
}

type crdbStartHelper struct {
type crdbInstallHelper struct {
c *SyncedCluster
r Cockroach
}

func (h *crdbStartHelper) startNode(
func (h *crdbInstallHelper) startNode(
nodeIdx int, extraArgs []string, vers *version.Version,
) (string, error) {
startCmd, err := h.generateStartCmd(nodeIdx, extraArgs, vers)
Expand All @@ -343,7 +353,7 @@ func (h *crdbStartHelper) startNode(
return strings.TrimSpace(string(out)), nil
}

func (h *crdbStartHelper) generateStartCmd(
func (h *crdbInstallHelper) generateStartCmd(
nodeIdx int, extraArgs []string, vers *version.Version,
) (string, error) {
args, err := h.generateStartArgs(nodeIdx, extraArgs, vers)
Expand Down Expand Up @@ -390,7 +400,7 @@ func (h *crdbStartHelper) generateStartCmd(
return cmd, nil
}

func (h *crdbStartHelper) generateStartArgs(
func (h *crdbInstallHelper) generateStartArgs(
nodeIdx int, extraArgs []string, vers *version.Version,
) ([]string, error) {
var args []string
Expand Down Expand Up @@ -491,7 +501,7 @@ func (h *crdbStartHelper) generateStartArgs(
return args, nil
}

func (h *crdbStartHelper) initializeCluster(nodeIdx int) (string, error) {
func (h *crdbInstallHelper) initializeCluster(nodeIdx int) (string, error) {
nodes := h.c.ServerNodes()
initCmd := h.generateInitCmd(nodeIdx)

Expand All @@ -508,7 +518,7 @@ func (h *crdbStartHelper) initializeCluster(nodeIdx int) (string, error) {
return strings.TrimSpace(string(out)), nil
}

func (h *crdbStartHelper) setClusterSettings(nodeIdx int) (string, error) {
func (h *crdbInstallHelper) setClusterSettings(nodeIdx int) (string, error) {
nodes := h.c.ServerNodes()
clusterSettingCmd := h.generateClusterSettingCmd(nodeIdx)

Expand All @@ -525,7 +535,7 @@ func (h *crdbStartHelper) setClusterSettings(nodeIdx int) (string, error) {
return strings.TrimSpace(string(out)), nil
}

func (h *crdbStartHelper) generateClusterSettingCmd(nodeIdx int) string {
func (h *crdbInstallHelper) generateClusterSettingCmd(nodeIdx int) string {
nodes := h.c.ServerNodes()
license := envutil.EnvOrDefaultString("COCKROACH_DEV_LICENSE", "")
if license == "" {
Expand Down Expand Up @@ -553,7 +563,7 @@ func (h *crdbStartHelper) generateClusterSettingCmd(nodeIdx int) string {
return clusterSettingCmd
}

func (h *crdbStartHelper) generateInitCmd(nodeIdx int) string {
func (h *crdbInstallHelper) generateInitCmd(nodeIdx int) string {
nodes := h.c.ServerNodes()

var initCmd string
Expand All @@ -572,7 +582,7 @@ func (h *crdbStartHelper) generateInitCmd(nodeIdx int) string {
return initCmd
}

func (h *crdbStartHelper) generateKeyCmd(nodeIdx int, extraArgs []string) string {
func (h *crdbInstallHelper) generateKeyCmd(nodeIdx int, extraArgs []string) string {
if !StartOpts.Encrypt {
return ""
}
Expand All @@ -594,13 +604,13 @@ func (h *crdbStartHelper) generateKeyCmd(nodeIdx int, extraArgs []string) string
return keyCmd
}

func (h *crdbStartHelper) useStartSingleNode(vers *version.Version) bool {
func (h *crdbInstallHelper) useStartSingleNode(vers *version.Version) bool {
return len(h.c.VMs) == 1 && vers.AtLeast(version.MustParse("v19.2.0"))
}

// distributeCerts, like the name suggests, distributes certs if it's a secure
// cluster and we're starting n1.
func (h *crdbStartHelper) distributeCerts() {
func (h *crdbInstallHelper) distributeCerts() {
for _, node := range h.c.ServerNodes() {
if node == 1 && h.c.Secure {
h.c.DistributeCerts()
Expand All @@ -609,7 +619,7 @@ func (h *crdbStartHelper) distributeCerts() {
}
}

func (h *crdbStartHelper) shouldAdvertisePublicIP() bool {
func (h *crdbInstallHelper) shouldAdvertisePublicIP() bool {
// If we're creating nodes that span VPC (e.g. AWS multi-region or
// multi-cloud), we'll tell the nodes to advertise their public IPs
// so that attaching nodes to the cluster Just Works.
Expand All @@ -621,7 +631,7 @@ func (h *crdbStartHelper) shouldAdvertisePublicIP() bool {
return false
}

func (h *crdbStartHelper) getEnvVars() string {
func (h *crdbInstallHelper) getEnvVars() string {
var buf strings.Builder
for _, v := range os.Environ() {
if strings.HasPrefix(v, "COCKROACH_") {
Expand All @@ -640,7 +650,7 @@ func (h *crdbStartHelper) getEnvVars() string {
return buf.String()
}

func (h *crdbStartHelper) run(nodeIdx int, cmd string) (string, error) {
func (h *crdbInstallHelper) run(nodeIdx int, cmd string) (string, error) {
nodes := h.c.ServerNodes()

sess, err := h.c.newSession(nodes[nodeIdx])
Expand Down
33 changes: 32 additions & 1 deletion pkg/cmd/roachprod/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ var (
adminurlIPs = false
useTreeDist = true
encrypt = false
skipInit = false
quiet = false
sig = 9
waitFlag = false
Expand Down Expand Up @@ -984,7 +985,9 @@ environment variables to the cockroach process.
` + tagHelp + `
The "start" command takes care of setting up the --join address and specifying
reasonable defaults for other flags. One side-effect of this convenience is
that node 1 is special and must be started for the cluster to be initialized.
that node 1 is special and if started, is used to auto-initialize the cluster.
The --skip-init flag can be used to avoid auto-initialization (which can then
separately be done using the "init" command).

If the COCKROACH_DEV_LICENSE environment variable is set the enterprise.license
cluster setting will be set to its value.
Expand Down Expand Up @@ -1035,6 +1038,31 @@ other signals.
}),
}

var initCmd = &cobra.Command{
Use: "init <cluster>",
Short: "initialize the cluster",
Long: `Initialize the cluster.

The "init" command bootstraps the cluster (using "cockroach init"). It also sets
default cluster settings. It's intended to be used in conjunction with
'roachprod start --skip-init'.
`,
Args: cobra.ExactArgs(1),
Run: wrap(func(cmd *cobra.Command, args []string) error {
clusterName, err := verifyClusterName(args[0])
if err != nil {
return err
}

c, err := newCluster(clusterName)
if err != nil {
return err
}
c.Init()
return nil
}),
}

var statusCmd = &cobra.Command{
Use: "status <cluster>",
Short: "retrieve the status of nodes in a cluster",
Expand Down Expand Up @@ -1558,6 +1586,7 @@ func main() {
monitorCmd,
startCmd,
stopCmd,
initCmd,
runCmd,
wipeCmd,
reformatCmd,
Expand Down Expand Up @@ -1754,6 +1783,8 @@ func main() {
&clusterType, "type", "t", clusterType, `cluster type ("cockroach" or "cassandra")`)
cmd.Flags().BoolVar(
&install.StartOpts.Encrypt, "encrypt", encrypt, "start nodes with encryption at rest turned on")
cmd.Flags().BoolVar(
&install.StartOpts.SkipInit, "skip-init", skipInit, "skip initializing the cluster")
fallthrough
case sqlCmd:
cmd.Flags().StringVarP(
Expand Down