Add platform for external provisioning

cmd/kola/options.go

@@ -39,7 +39,7 @@ var (
 	kolaOffering       string
 	defaultTargetBoard = sdk.DefaultBoard()
 	kolaArchitectures  = []string{"amd64"}
-	kolaPlatforms      = []string{"aws", "azure", "do", "esx", "gce", "openstack", "packet", "qemu", "qemu-unpriv"}
+	kolaPlatforms      = []string{"aws", "azure", "do", "esx", "external", "gce", "openstack", "packet", "qemu", "qemu-unpriv"}
 	kolaDistros        = []string{"cl", "fcos", "rhcos"}
 	kolaChannels       = []string{"alpha", "beta", "stable", "edge", "lts"}
 	kolaOfferings      = []string{"basic", "pro"}
@@ -128,6 +128,15 @@ func init() {
 	sv(&kola.ESXOptions.FirstStaticIpPrivate, "esx-first-static-ip-private", "", "First available private IP (only needed for static IP addresses)")
 	root.PersistentFlags().IntVarP(&kola.ESXOptions.StaticSubnetSize, "esx-subnet-size", "", 0, "Subnet size (only needed for static IP addresses)")
 
+	// external-specific options
+	sv(&kola.ExternalOptions.ManagementUser, "external-user", "", "External platform management SSH user")
+	sv(&kola.ExternalOptions.ManagementPassword, "external-password", "", "External platform management SSH password")
+	sv(&kola.ExternalOptions.ManagementHost, "external-host", "", "External platform management SSH host in the format HOST:PORT")
+	sv(&kola.ExternalOptions.ManagementSocks, "external-socks", "", "External platform management SSH via SOCKS5 proxy in the format HOST:PORT (optional)")
+	sv(&kola.ExternalOptions.ProvisioningCmds, "external-provisioning-cmds", "", "External platform provisioning commands ran on management SSH host. Has access to variable USERDATA with ignition config (can serve it via pxe http server for ignition.config.url or use as contents of FILE in 'flatcar-install -i FILE'). Note: It should mask sshd.(service|socket) for any booted PXE installer, and handle setting to boot from disk, as well as finding a free device and print its IP address as sole stdout content.")
+	sv(&kola.ExternalOptions.SerialConsoleCmd, "external-serial-console-cmd", "", "External platform serial console attach command ran on management SSH host. Has access to the variable IPADDR to identify the node.")
+	sv(&kola.ExternalOptions.DeprovisioningCmds, "external-deprovisioning-cmds", "", "External platform deprovisioning commands ran on management SSH host. Has access to the variable IPADDR to identify the node.")
+
 	// gce-specific options
 	sv(&kola.GCEOptions.Image, "gce-image", "projects/coreos-cloud/global/images/family/coreos-alpha", "GCE image, full api endpoints names are accepted if resource is in a different project")
 	sv(&kola.GCEOptions.Project, "gce-project", "flatcar-212911", "GCE project name")
@@ -178,6 +187,7 @@ func syncOptions() error {
 	kola.OpenStackOptions.Board = board
 	kola.GCEOptions.Board = board
 	kola.ESXOptions.Board = board
+	kola.ExternalOptions.Board = board
 	kola.DOOptions.Board = board
 	kola.AzureOptions.Board = board
 	kola.AWSOptions.Board = board

kola/harness.go

@@ -48,6 +48,7 @@ import (
 	"github.com/coreos/mantle/platform/machine/azure"
 	"github.com/coreos/mantle/platform/machine/do"
 	"github.com/coreos/mantle/platform/machine/esx"
+	"github.com/coreos/mantle/platform/machine/external"
 	"github.com/coreos/mantle/platform/machine/gcloud"
 	"github.com/coreos/mantle/platform/machine/openstack"
 	"github.com/coreos/mantle/platform/machine/packet"
@@ -64,6 +65,7 @@ var (
 	AzureOptions     = azureapi.Options{Options: &Options}     // glue to set platform options from main
 	DOOptions        = doapi.Options{Options: &Options}        // glue to set platform options from main
 	ESXOptions       = esxapi.Options{Options: &Options}       // glue to set platform options from main
+	ExternalOptions  = external.Options{Options: &Options}     // glue to set platform options from main
 	GCEOptions       = gcloudapi.Options{Options: &Options}    // glue to set platform options from main
 	OpenStackOptions = openstackapi.Options{Options: &Options} // glue to set platform options from main
 	PacketOptions    = packetapi.Options{Options: &Options}    // glue to set platform options from main
@@ -174,6 +176,8 @@ func NewFlight(pltfrm string) (flight platform.Flight, err error) {
 		flight, err = do.NewFlight(&DOOptions)
 	case "esx":
 		flight, err = esx.NewFlight(&ESXOptions)
+	case "external":
+		flight, err = external.NewFlight(&ExternalOptions)
 	case "gce":
 		flight, err = gcloud.NewFlight(&GCEOptions)
 	case "openstack":

network/nsdialer.go

@@ -31,7 +31,7 @@ type NsDialer struct {
 func NewNsDialer(ns netns.NsHandle) *NsDialer {
 	return &NsDialer{
 		RetryDialer: RetryDialer{
-			Dialer: net.Dialer{
+			Dialer: &net.Dialer{
 				Timeout:   DefaultTimeout,
 				KeepAlive: DefaultKeepAlive,
 			},

network/retrydialer.go

@@ -33,14 +33,14 @@ const (
 // RetryDialer is intended to timeout quickly and retry connecting instead
 // of just failing. Particularly useful for waiting on a booting machine.
 type RetryDialer struct {
-	net.Dialer
+	Dialer
 	Retries int
 }
 
 // NewRetryDialer initializes a RetryDialer with reasonable default settings.
 func NewRetryDialer() *RetryDialer {
 	return &RetryDialer{
-		Dialer: net.Dialer{
+		Dialer: &net.Dialer{
 			Timeout:   DefaultTimeout,
 			KeepAlive: DefaultKeepAlive,
 		},

platform/machine/external/cluster.go

@@ -0,0 +1,240 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package external
+
+import (
+	"crypto/rand"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/crypto/ssh"
+
+	"github.com/coreos/mantle/platform"
+	"github.com/coreos/mantle/platform/conf"
+)
+
+type cluster struct {
+	*platform.BaseCluster
+	flight *flight
+}
+
+func (pc *cluster) NewMachine(userdata *conf.UserData) (platform.Machine, error) {
+	conf, err := pc.RenderUserData(userdata, map[string]string{
+		"$public_ipv4":  "${COREOS_CUSTOM_PUBLIC_IPV4}",
+		"$private_ipv4": "${COREOS_CUSTOM_PRIVATE_IPV4}",
+	})
+	if err != nil {
+		return nil, err
+	}
+	// This assumes that private and public IP addresses are the same (i.e., no public IP addr) on the interface that has the default route
+	conf.AddSystemdUnit("coreos-metadata.service", `[Unit]
+Description=Custom metadata agent
+After=nss-lookup.target
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+Environment=OUTPUT=/run/metadata/flatcar
+ExecStart=/usr/bin/mkdir --parent /run/metadata
+ExecStart=/usr/bin/bash -c 'echo "COREOS_CUSTOM_PRIVATE_IPV4=$(ip addr show $(ip route get 1 | head -n 1 | cut -d ' ' -f 5) | grep -m 1 -Po "inet \K[\d.]+")\nCOREOS_CUSTOM_PUBLIC_IPV4=$(ip addr show $(ip route get 1 | head -n 1 | cut -d ' ' -f 5) | grep -m 1 -Po "inet \K[\d.]+")" > ${OUTPUT}'
+ExecStartPost=/usr/bin/ln -fs /run/metadata/flatcar /run/metadata/coreos
+`, false)
+
+	var cons *console
+	var pcons Console // need a nil interface value if unused
+	var ipAddr string
+	// Do not shadow assignments to err (i.e., use a, err := something) in the for loop
+	// because the "continue" case needs to access the previous error to return it when the
+	// maximal number of retries is reached or to print it at the beginning of the loop.
+	for retry := 0; retry <= 2; retry++ {
+		if err != nil {
+			plog.Warningf("Retrying to provision a machine after error: %q", err)
+		}
+		// Stream the console somewhere temporary until we have a machine ID
+		b := make([]byte, 5)
+		rand.Read(b)
+		randName := fmt.Sprintf("%x", b)
+		consolePath := filepath.Join(pc.RuntimeConf().OutputDir, "console-"+pc.Name()[0:13]+"-"+randName+".txt")
+		var f *os.File
+		f, err = os.OpenFile(consolePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
+		if err != nil {
+			return nil, err
+		}
+		cons = &console{
+			pc:   pc,
+			f:    f,
+			done: make(chan interface{}),
+		}
+		pcons = cons
+
+		// CreateDevice unconditionally closes console when done with it
+		ipAddr, err = pc.createDevice(conf, pcons)
+		if err != nil {
+			continue // provisioning error
+		}
+
+		mach := &machine{
+			cluster: pc,
+			ipAddr:  ipAddr,
+			console: cons,
+			rand:    randName,
+		}
+
+		dir := filepath.Join(pc.RuntimeConf().OutputDir, mach.ID())
+		if err = os.Mkdir(dir, 0777); err != nil {
+			mach.Destroy()
+			return nil, err
+		}
+
+		if cons != nil {
+			if err = os.Rename(consolePath, filepath.Join(dir, "console.txt")); err != nil {
+				mach.Destroy()
+				return nil, err
+			}
+		}
+
+		confPath := filepath.Join(dir, "user-data")
+		if err = conf.WriteFile(confPath); err != nil {
+			mach.Destroy()
+			return nil, err
+		}
+
+		if mach.journal, err = platform.NewJournal(dir); err != nil {
+			mach.Destroy()
+			return nil, err
+		}
+
+		plog.Infof("Starting machine %v", mach.ID())
+		if err = platform.StartMachine(mach, mach.journal); err != nil {
+			mach.Destroy()
+			continue // provisioning error
+		}
+
+		pc.AddMach(mach)
+
+		return mach, nil
+
+	}
+
+	return nil, err
+}
+
+func setEnvCmd(varname, content string) string {
+	quoted := strings.ReplaceAll(content, `'`, `'"'"'`)
+	// include the final ; for concatenation with a following command
+	return varname + `='` + quoted + `';`
+}
+
+func (pc *cluster) createDevice(conf *conf.Conf, console Console) (string, error) {
+	plog.Info("Creating machine")
+	consoleStarted := false
+	defer func() {
+		if console != nil && !consoleStarted {
+			console.Close()
+		}
+	}()
+
+	userdata := conf.String()
+	session, err := pc.flight.ManagementSSHClient.NewSession()
+	if err != nil {
+		return "", err
+	}
+	defer session.Close()
+	output, err := session.Output(setEnvCmd("USERDATA", userdata) + pc.flight.ExternalOptions.ProvisioningCmds)
+	if err != nil {
+		return "", err
+	}
+	ipAddr := strings.TrimSpace(string(output))
+	if net.ParseIP(ipAddr) == nil {
+		return "", fmt.Errorf("script output %q is not a valid IP address", ipAddr)
+	}
+	plog.Infof("Got IP address %v", ipAddr)
+	if console != nil {
+		err := pc.startConsole(ipAddr, console)
+		// console will be closed in any case
+		consoleStarted = true
+		if err != nil {
+			err2 := pc.deleteDevice(ipAddr)
+			if err2 != nil {
+				return "", fmt.Errorf("couldn't delete device %s after error %q: %v", ipAddr, err, err2)
+			}
+			return "", err
+		}
+	}
+	return ipAddr, nil
+}
+
+func (pc *cluster) deleteDevice(ipAddr string) error {
+	plog.Infof("Deleting machine %v", ipAddr)
+	session, err := pc.flight.ManagementSSHClient.NewSession()
+	if err != nil {
+		return err
+	}
+	defer session.Close()
+	err = session.Run(setEnvCmd("IPADDR", ipAddr) + pc.flight.ExternalOptions.DeprovisioningCmds)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (pc *cluster) startConsole(ipAddr string, console Console) error {
+	plog.Infof("Attaching serial console for %v", ipAddr)
+	ready := make(chan error)
+
+	runner := func() error {
+		defer console.Close()
+		session, err := pc.flight.ManagementSSHClient.NewSession()
+		if err != nil {
+			return fmt.Errorf("couldn't create SSH session for %s console: %v", ipAddr, err)
+		}
+		defer session.Close()
+
+		reader, writer := io.Pipe()
+		defer writer.Close()
+
+		session.Stdin = reader
+		session.Stdout = console
+		if err := session.Start(setEnvCmd("IPADDR", ipAddr) + pc.flight.ExternalOptions.SerialConsoleCmd); err != nil {
+			return fmt.Errorf("couldn't start provided serial attach command for %s console: %v", ipAddr, err)
+		}
+
+		// cause startConsole to return
+		ready <- nil
+
+		err = session.Wait()
+		_, ok := err.(*ssh.ExitMissingError)
+		if err != nil && !ok {
+			plog.Errorf("%s console session failed: %v", ipAddr, err)
+		}
+		return nil
+	}
+	go func() {
+		err := runner()
+		if err != nil {
+			ready <- err
+		}
+	}()
+
+	return <-ready
+}
+
+func (pc *cluster) Destroy() {
+	pc.BaseCluster.Destroy()
+	pc.flight.DelCluster(pc)
+}

platform/machine/external/console.go

@@ -0,0 +1,45 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package external
+
+import (
+	"bytes"
+	"io"
+	"os"
+)
+
+type Console interface {
+	io.WriteCloser
+}
+
+type console struct {
+	pc   *cluster
+	f    *os.File
+	buf  bytes.Buffer
+	done chan interface{}
+}
+
+func (c *console) Write(p []byte) (int, error) {
+	c.buf.Write(p)
+	return c.f.Write(p)
+}
+
+func (c *console) Close() error {
+	close(c.done)
+	return c.f.Close()
+}
+
+func (c *console) Output() string {
+	<-c.done
+	return c.buf.String()
+}