-
Notifications
You must be signed in to change notification settings - Fork 13
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add platform for external provisioning
The existing platforms have their provisioning logic in Go, using cloud-vendor API libraries or a hardcoded QEMU setup. Following this approach it's difficult to add a custom platform for, e.g., a PXE and IPMI environment. Introduce a new generic platform which is based on external provisioning commands. The kola user can thus provide shell commands for provisioning, console attachment, and deprovisioning, be it using ipmitool and a custom PXE setup or invoking Terraform or any other tools to create a virtual machine. For now the commands are run via SSH because this is the current use case but local execution can be added when needed. It's also possible to let the SSH session be created through a SOCKS5 proxy for, e.g., having the management SSH host on a Hidden Tor Service. SSH login is done with a password, but here, too, using keys can be added when needed. An example invocation is: ./kola run -d -k --board=arm64-usr --platform=external --external-user=core "--external-password=$(cat external-password)" "--external-provisioning-cmds=$(echo BASE_URL=http://alpha.release.flatcar-linux.net/arm64-usr/current; cat external-provisioning-cmds)" "--external-serial-console-cmd=$(cat external-serial-console-cmd)" "--external-deprovisioning-cmds=$(cat external-deprovisioning-cmds)" --external-socks=127.0.0.1:9050 --external-host=myonionserver.onion:22 cl.basic And the content of the "external-*" scripts can be found in the flatcar-scripts/jenkins/kola/ folder.
- Loading branch information
Showing
16 changed files
with
1,456 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,239 @@ | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package external | ||
|
|
||
| import ( | ||
| "crypto/rand" | ||
| "fmt" | ||
| "io" | ||
| "os" | ||
| "path/filepath" | ||
| "strings" | ||
|
|
||
| "golang.org/x/crypto/ssh" | ||
|
|
||
| "github.com/coreos/mantle/platform" | ||
| "github.com/coreos/mantle/platform/conf" | ||
| ) | ||
|
|
||
| type cluster struct { | ||
| *platform.BaseCluster | ||
| flight *flight | ||
| } | ||
|
|
||
| func (pc *cluster) NewMachine(userdata *conf.UserData) (platform.Machine, error) { | ||
| conf, err := pc.RenderUserData(userdata, map[string]string{ | ||
| "$public_ipv4": "${COREOS_CUSTOM_PUBLIC_IPV4}", | ||
| "$private_ipv4": "${COREOS_CUSTOM_PRIVATE_IPV4}", | ||
| }) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| // This assumes that private and public IP addresses are the same (i.e., no public IP addr) on the interface that has the default route | ||
| conf.AddSystemdUnit("coreos-metadata.service", `[Unit] | ||
| Description=Custom metadata agent | ||
| After=nss-lookup.target | ||
| After=network-online.target | ||
| Wants=network-online.target | ||
| [Service] | ||
| Type=oneshot | ||
| Environment=OUTPUT=/run/metadata/flatcar | ||
| ExecStart=/usr/bin/mkdir --parent /run/metadata | ||
| ExecStart=/usr/bin/bash -c 'echo "COREOS_CUSTOM_PRIVATE_IPV4=$(ip addr show $(ip route get 1 | head -n 1 | cut -d ' ' -f 5) grep -m 1 -Po "inet \K[\d.]+")\nCOREOS_CUSTOM_PUBLIC_IPV4=$(ip addr show $(ip route get 1 | head -n 1 | cut -d ' ' -f 5) | grep -m 1 -Po "inet \K[\d.]+")" > ${OUTPUT}' | ||
| ExecStartPost=/usr/bin/ln -fs /run/metadata/flatcar /run/metadata/coreos | ||
| `, false) | ||
|
|
||
| var cons *console | ||
| var pcons Console // need a nil interface value if unused | ||
| var ipAddr string | ||
| // Do not shadow assignments to err (i.e., use a, err := something) in the for loop | ||
| // because the "continue" case needs to access the previous error to return it when the | ||
| // maximal number of retries is reached or to print it at the beginning of the loop. | ||
| for retry := 0; retry <= 2; retry++ { | ||
| // Stream the console somewhere temporary until we have a machine ID | ||
| b := make([]byte, 5) | ||
| rand.Read(b) | ||
| randName := fmt.Sprintf("%x", b) | ||
| consolePath := filepath.Join(pc.RuntimeConf().OutputDir, "console-"+pc.Name()[0:13]+"-"+randName+".txt") | ||
| if err != nil { | ||
| plog.Warningf("Retrying to provision a machine after error: %q", err) | ||
| err = os.Remove(consolePath) | ||
| if err != nil && !os.IsNotExist(err) { | ||
| return nil, err | ||
| } | ||
| } | ||
| var f *os.File | ||
| f, err = os.OpenFile(consolePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| cons = &console{ | ||
| pc: pc, | ||
| f: f, | ||
| done: make(chan interface{}), | ||
| } | ||
| pcons = cons | ||
|
|
||
| // CreateDevice unconditionally closes console when done with it | ||
| ipAddr, err = pc.createDevice(conf, pcons) | ||
| if err != nil { | ||
| continue // provisioning error | ||
| } | ||
|
|
||
| mach := &machine{ | ||
| cluster: pc, | ||
| ipAddr: ipAddr, | ||
| console: cons, | ||
| rand: randName, | ||
| } | ||
|
|
||
| dir := filepath.Join(pc.RuntimeConf().OutputDir, mach.ID()) | ||
| if err = os.Mkdir(dir, 0777); err != nil { | ||
| mach.Destroy() | ||
| return nil, err | ||
| } | ||
|
|
||
| if cons != nil { | ||
| if err = os.Rename(consolePath, filepath.Join(dir, "console.txt")); err != nil { | ||
| mach.Destroy() | ||
| return nil, err | ||
| } | ||
| } | ||
|
|
||
| confPath := filepath.Join(dir, "user-data") | ||
| if err = conf.WriteFile(confPath); err != nil { | ||
| mach.Destroy() | ||
| return nil, err | ||
| } | ||
|
|
||
| if mach.journal, err = platform.NewJournal(dir); err != nil { | ||
| mach.Destroy() | ||
| return nil, err | ||
| } | ||
|
|
||
| plog.Infof("Starting machine %v", mach.ID()) | ||
| if err = platform.StartMachine(mach, mach.journal); err != nil { | ||
| mach.Destroy() | ||
| continue // provisioning error | ||
| } | ||
|
|
||
| pc.AddMach(mach) | ||
|
|
||
| return mach, nil | ||
|
|
||
| } | ||
|
|
||
| return nil, err | ||
| } | ||
|
|
||
| func setEnvCmd(varname, content string) string { | ||
| quoted := strings.ReplaceAll(content, `'`, `'"'"'`) | ||
| // include the final ; for concatenation with a following command | ||
| return varname + `='` + quoted + `';` | ||
| } | ||
|
|
||
| func (pc *cluster) createDevice(conf *conf.Conf, console Console) (string, error) { | ||
| plog.Info("Creating machine") | ||
| consoleStarted := false | ||
| defer func() { | ||
| if console != nil && !consoleStarted { | ||
| console.Close() | ||
| } | ||
| }() | ||
|
|
||
| userdata := conf.String() | ||
| session, err := pc.flight.ManagementSSHClient.NewSession() | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| defer session.Close() | ||
| output, err := session.Output(setEnvCmd("USERDATA", userdata) + pc.flight.ExternalOptions.ProvisioningCmds) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| ipAddr := strings.TrimSpace(string(output)) | ||
| plog.Infof("Got IP address %v", ipAddr) | ||
| if console != nil { | ||
| err := pc.startConsole(ipAddr, console) | ||
| consoleStarted = true | ||
| if err != nil { | ||
| err2 := pc.deleteDevice(ipAddr) | ||
| if err2 != nil { | ||
| return "", fmt.Errorf("couldn't delete device %s after error %q: %v", ipAddr, err, err2) | ||
| } | ||
| return "", err | ||
| } | ||
| } | ||
| return ipAddr, nil | ||
| } | ||
|
|
||
| func (pc *cluster) deleteDevice(ipAddr string) error { | ||
| plog.Infof("Deleting machine %v", ipAddr) | ||
| session, err := pc.flight.ManagementSSHClient.NewSession() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer session.Close() | ||
| err = session.Run(setEnvCmd("IPADDR", ipAddr) + pc.flight.ExternalOptions.DeprovisioningCmds) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func (pc *cluster) startConsole(ipAddr string, console Console) error { | ||
| plog.Infof("Attaching serial console for %v", ipAddr) | ||
| ready := make(chan error) | ||
|
|
||
| runner := func() error { | ||
| defer console.Close() | ||
| session, err := pc.flight.ManagementSSHClient.NewSession() | ||
| if err != nil { | ||
| return fmt.Errorf("couldn't create SSH session for %s console: %v", ipAddr, err) | ||
| } | ||
| defer session.Close() | ||
|
|
||
| reader, writer := io.Pipe() | ||
| defer writer.Close() | ||
|
|
||
| session.Stdin = reader | ||
| session.Stdout = console | ||
| if err := session.Start(setEnvCmd("IPADDR", ipAddr) + pc.flight.ExternalOptions.SerialConsoleCmd); err != nil { | ||
| return fmt.Errorf("couldn't start provided serial attach command for %s console: %v", ipAddr, err) | ||
| } | ||
|
|
||
| // cause startConsole to return | ||
| ready <- nil | ||
|
|
||
| err = session.Wait() | ||
| _, ok := err.(*ssh.ExitMissingError) | ||
| if err != nil && !ok { | ||
| plog.Errorf("%s console session failed: %v", ipAddr, err) | ||
| } | ||
| return nil | ||
| } | ||
| go func() { | ||
| err := runner() | ||
| if err != nil { | ||
| ready <- err | ||
| } | ||
| }() | ||
|
|
||
| return <-ready | ||
| } | ||
|
|
||
| func (pc *cluster) Destroy() { | ||
| pc.BaseCluster.Destroy() | ||
| pc.flight.DelCluster(pc) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package external | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "io" | ||
| "os" | ||
| ) | ||
|
|
||
| type Console interface { | ||
| io.WriteCloser | ||
| } | ||
|
|
||
| type console struct { | ||
| pc *cluster | ||
| f *os.File | ||
| buf bytes.Buffer | ||
| done chan interface{} | ||
| } | ||
|
|
||
| func (c *console) Write(p []byte) (int, error) { | ||
| c.buf.Write(p) | ||
| return c.f.Write(p) | ||
| } | ||
|
|
||
| func (c *console) Close() error { | ||
| close(c.done) | ||
| return c.f.Close() | ||
| } | ||
|
|
||
| func (c *console) Output() string { | ||
| <-c.done | ||
| return c.buf.String() | ||
| } |
Oops, something went wrong.