diff --git a/cmd/kola/options.go b/cmd/kola/options.go index 04da1c020..ac405a457 100644 --- a/cmd/kola/options.go +++ b/cmd/kola/options.go @@ -39,7 +39,7 @@ var ( kolaOffering string defaultTargetBoard = sdk.DefaultBoard() kolaArchitectures = []string{"amd64"} - kolaPlatforms = []string{"aws", "azure", "do", "esx", "gce", "openstack", "packet", "qemu", "qemu-unpriv"} + kolaPlatforms = []string{"aws", "azure", "do", "esx", "external", "gce", "openstack", "packet", "qemu", "qemu-unpriv"} kolaDistros = []string{"cl", "fcos", "rhcos"} kolaChannels = []string{"alpha", "beta", "stable", "edge", "lts"} kolaOfferings = []string{"basic", "pro"} @@ -128,6 +128,15 @@ func init() { sv(&kola.ESXOptions.FirstStaticIpPrivate, "esx-first-static-ip-private", "", "First available private IP (only needed for static IP addresses)") root.PersistentFlags().IntVarP(&kola.ESXOptions.StaticSubnetSize, "esx-subnet-size", "", 0, "Subnet size (only needed for static IP addresses)") + // external-specific options + sv(&kola.ExternalOptions.ManagementUser, "external-user", "", "External platform management SSH user") + sv(&kola.ExternalOptions.ManagementPassword, "external-password", "", "External platform management SSH password") + sv(&kola.ExternalOptions.ManagementHost, "external-host", "", "External platform management SSH host in the format HOST:PORT") + sv(&kola.ExternalOptions.ManagementSocks, "external-socks", "", "External platform management SSH via SOCKS5 proxy in the format HOST:PORT (optional)") + sv(&kola.ExternalOptions.ProvisioningCmds, "external-provisioning-cmds", "", "External platform provisioning commands ran on management SSH host. Has access to variable USERDATA with ignition config (can serve it via pxe http server for ignition.config.url or use as contents of FILE in 'flatcar-install -i FILE'). Note: It should mask sshd.(service|socket) for any booted PXE installer, and handle setting to boot from disk, as well as finding a free device and print its IP address as sole stdout content.") + sv(&kola.ExternalOptions.SerialConsoleCmd, "external-serial-console-cmd", "", "External platform serial console attach command ran on management SSH host. Has access to the variable IPADDR to identify the node.") + sv(&kola.ExternalOptions.DeprovisioningCmds, "external-deprovisioning-cmds", "", "External platform deprovisioning commands ran on management SSH host. Has access to the variable IPADDR to identify the node.") + // gce-specific options sv(&kola.GCEOptions.Image, "gce-image", "projects/coreos-cloud/global/images/family/coreos-alpha", "GCE image, full api endpoints names are accepted if resource is in a different project") sv(&kola.GCEOptions.Project, "gce-project", "flatcar-212911", "GCE project name") @@ -178,6 +187,7 @@ func syncOptions() error { kola.OpenStackOptions.Board = board kola.GCEOptions.Board = board kola.ESXOptions.Board = board + kola.ExternalOptions.Board = board kola.DOOptions.Board = board kola.AzureOptions.Board = board kola.AWSOptions.Board = board diff --git a/kola/harness.go b/kola/harness.go index 6ef30744b..88f9b01f3 100644 --- a/kola/harness.go +++ b/kola/harness.go @@ -48,6 +48,7 @@ import ( "github.com/coreos/mantle/platform/machine/azure" "github.com/coreos/mantle/platform/machine/do" "github.com/coreos/mantle/platform/machine/esx" + "github.com/coreos/mantle/platform/machine/external" "github.com/coreos/mantle/platform/machine/gcloud" "github.com/coreos/mantle/platform/machine/openstack" "github.com/coreos/mantle/platform/machine/packet" @@ -64,6 +65,7 @@ var ( AzureOptions = azureapi.Options{Options: &Options} // glue to set platform options from main DOOptions = doapi.Options{Options: &Options} // glue to set platform options from main ESXOptions = esxapi.Options{Options: &Options} // glue to set platform options from main + ExternalOptions = external.Options{Options: &Options} // glue to set platform options from main GCEOptions = gcloudapi.Options{Options: &Options} // glue to set platform options from main OpenStackOptions = openstackapi.Options{Options: &Options} // glue to set platform options from main PacketOptions = packetapi.Options{Options: &Options} // glue to set platform options from main @@ -174,6 +176,8 @@ func NewFlight(pltfrm string) (flight platform.Flight, err error) { flight, err = do.NewFlight(&DOOptions) case "esx": flight, err = esx.NewFlight(&ESXOptions) + case "external": + flight, err = external.NewFlight(&ExternalOptions) case "gce": flight, err = gcloud.NewFlight(&GCEOptions) case "openstack": diff --git a/network/nsdialer.go b/network/nsdialer.go index a25e66cdf..99609fdff 100644 --- a/network/nsdialer.go +++ b/network/nsdialer.go @@ -31,7 +31,7 @@ type NsDialer struct { func NewNsDialer(ns netns.NsHandle) *NsDialer { return &NsDialer{ RetryDialer: RetryDialer{ - Dialer: net.Dialer{ + Dialer: &net.Dialer{ Timeout: DefaultTimeout, KeepAlive: DefaultKeepAlive, }, diff --git a/network/retrydialer.go b/network/retrydialer.go index f20d5fe5f..a3b3a3720 100644 --- a/network/retrydialer.go +++ b/network/retrydialer.go @@ -33,14 +33,14 @@ const ( // RetryDialer is intended to timeout quickly and retry connecting instead // of just failing. Particularly useful for waiting on a booting machine. type RetryDialer struct { - net.Dialer + Dialer Retries int } // NewRetryDialer initializes a RetryDialer with reasonable default settings. func NewRetryDialer() *RetryDialer { return &RetryDialer{ - Dialer: net.Dialer{ + Dialer: &net.Dialer{ Timeout: DefaultTimeout, KeepAlive: DefaultKeepAlive, }, diff --git a/platform/machine/external/cluster.go b/platform/machine/external/cluster.go new file mode 100644 index 000000000..df44ba07c --- /dev/null +++ b/platform/machine/external/cluster.go @@ -0,0 +1,240 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package external + +import ( + "crypto/rand" + "fmt" + "io" + "net" + "os" + "path/filepath" + "strings" + + "golang.org/x/crypto/ssh" + + "github.com/coreos/mantle/platform" + "github.com/coreos/mantle/platform/conf" +) + +type cluster struct { + *platform.BaseCluster + flight *flight +} + +func (pc *cluster) NewMachine(userdata *conf.UserData) (platform.Machine, error) { + conf, err := pc.RenderUserData(userdata, map[string]string{ + "$public_ipv4": "${COREOS_CUSTOM_PUBLIC_IPV4}", + "$private_ipv4": "${COREOS_CUSTOM_PRIVATE_IPV4}", + }) + if err != nil { + return nil, err + } + // This assumes that private and public IP addresses are the same (i.e., no public IP addr) on the interface that has the default route + conf.AddSystemdUnit("coreos-metadata.service", `[Unit] +Description=Custom metadata agent +After=nss-lookup.target +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +Environment=OUTPUT=/run/metadata/flatcar +ExecStart=/usr/bin/mkdir --parent /run/metadata +ExecStart=/usr/bin/bash -c 'echo "COREOS_CUSTOM_PRIVATE_IPV4=$(ip addr show $(ip route get 1 | head -n 1 | cut -d ' ' -f 5) | grep -m 1 -Po "inet \K[\d.]+")\nCOREOS_CUSTOM_PUBLIC_IPV4=$(ip addr show $(ip route get 1 | head -n 1 | cut -d ' ' -f 5) | grep -m 1 -Po "inet \K[\d.]+")" > ${OUTPUT}' +ExecStartPost=/usr/bin/ln -fs /run/metadata/flatcar /run/metadata/coreos +`, false) + + var cons *console + var pcons Console // need a nil interface value if unused + var ipAddr string + // Do not shadow assignments to err (i.e., use a, err := something) in the for loop + // because the "continue" case needs to access the previous error to return it when the + // maximal number of retries is reached or to print it at the beginning of the loop. + for retry := 0; retry <= 2; retry++ { + if err != nil { + plog.Warningf("Retrying to provision a machine after error: %q", err) + } + // Stream the console somewhere temporary until we have a machine ID + b := make([]byte, 5) + rand.Read(b) + randName := fmt.Sprintf("%x", b) + consolePath := filepath.Join(pc.RuntimeConf().OutputDir, "console-"+pc.Name()[0:13]+"-"+randName+".txt") + var f *os.File + f, err = os.OpenFile(consolePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) + if err != nil { + return nil, err + } + cons = &console{ + pc: pc, + f: f, + done: make(chan interface{}), + } + pcons = cons + + // CreateDevice unconditionally closes console when done with it + ipAddr, err = pc.createDevice(conf, pcons) + if err != nil { + continue // provisioning error + } + + mach := &machine{ + cluster: pc, + ipAddr: ipAddr, + console: cons, + rand: randName, + } + + dir := filepath.Join(pc.RuntimeConf().OutputDir, mach.ID()) + if err = os.Mkdir(dir, 0777); err != nil { + mach.Destroy() + return nil, err + } + + if cons != nil { + if err = os.Rename(consolePath, filepath.Join(dir, "console.txt")); err != nil { + mach.Destroy() + return nil, err + } + } + + confPath := filepath.Join(dir, "user-data") + if err = conf.WriteFile(confPath); err != nil { + mach.Destroy() + return nil, err + } + + if mach.journal, err = platform.NewJournal(dir); err != nil { + mach.Destroy() + return nil, err + } + + plog.Infof("Starting machine %v", mach.ID()) + if err = platform.StartMachine(mach, mach.journal); err != nil { + mach.Destroy() + continue // provisioning error + } + + pc.AddMach(mach) + + return mach, nil + + } + + return nil, err +} + +func setEnvCmd(varname, content string) string { + quoted := strings.ReplaceAll(content, `'`, `'"'"'`) + // include the final ; for concatenation with a following command + return varname + `='` + quoted + `';` +} + +func (pc *cluster) createDevice(conf *conf.Conf, console Console) (string, error) { + plog.Info("Creating machine") + consoleStarted := false + defer func() { + if console != nil && !consoleStarted { + console.Close() + } + }() + + userdata := conf.String() + session, err := pc.flight.ManagementSSHClient.NewSession() + if err != nil { + return "", err + } + defer session.Close() + output, err := session.Output(setEnvCmd("USERDATA", userdata) + pc.flight.ExternalOptions.ProvisioningCmds) + if err != nil { + return "", err + } + ipAddr := strings.TrimSpace(string(output)) + if net.ParseIP(ipAddr) == nil { + return "", fmt.Errorf("script output %q is not a valid IP address", ipAddr) + } + plog.Infof("Got IP address %v", ipAddr) + if console != nil { + err := pc.startConsole(ipAddr, console) + // console will be closed in any case + consoleStarted = true + if err != nil { + err2 := pc.deleteDevice(ipAddr) + if err2 != nil { + return "", fmt.Errorf("couldn't delete device %s after error %q: %v", ipAddr, err, err2) + } + return "", err + } + } + return ipAddr, nil +} + +func (pc *cluster) deleteDevice(ipAddr string) error { + plog.Infof("Deleting machine %v", ipAddr) + session, err := pc.flight.ManagementSSHClient.NewSession() + if err != nil { + return err + } + defer session.Close() + err = session.Run(setEnvCmd("IPADDR", ipAddr) + pc.flight.ExternalOptions.DeprovisioningCmds) + if err != nil { + return err + } + return nil +} + +func (pc *cluster) startConsole(ipAddr string, console Console) error { + plog.Infof("Attaching serial console for %v", ipAddr) + ready := make(chan error) + + runner := func() error { + defer console.Close() + session, err := pc.flight.ManagementSSHClient.NewSession() + if err != nil { + return fmt.Errorf("couldn't create SSH session for %s console: %v", ipAddr, err) + } + defer session.Close() + + reader, writer := io.Pipe() + defer writer.Close() + + session.Stdin = reader + session.Stdout = console + if err := session.Start(setEnvCmd("IPADDR", ipAddr) + pc.flight.ExternalOptions.SerialConsoleCmd); err != nil { + return fmt.Errorf("couldn't start provided serial attach command for %s console: %v", ipAddr, err) + } + + // cause startConsole to return + ready <- nil + + err = session.Wait() + _, ok := err.(*ssh.ExitMissingError) + if err != nil && !ok { + plog.Errorf("%s console session failed: %v", ipAddr, err) + } + return nil + } + go func() { + err := runner() + if err != nil { + ready <- err + } + }() + + return <-ready +} + +func (pc *cluster) Destroy() { + pc.BaseCluster.Destroy() + pc.flight.DelCluster(pc) +} diff --git a/platform/machine/external/console.go b/platform/machine/external/console.go new file mode 100644 index 000000000..5e55585e2 --- /dev/null +++ b/platform/machine/external/console.go @@ -0,0 +1,45 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package external + +import ( + "bytes" + "io" + "os" +) + +type Console interface { + io.WriteCloser +} + +type console struct { + pc *cluster + f *os.File + buf bytes.Buffer + done chan interface{} +} + +func (c *console) Write(p []byte) (int, error) { + c.buf.Write(p) + return c.f.Write(p) +} + +func (c *console) Close() error { + close(c.done) + return c.f.Close() +} + +func (c *console) Output() string { + <-c.done + return c.buf.String() +} diff --git a/platform/machine/external/flight.go b/platform/machine/external/flight.go new file mode 100644 index 000000000..05ddcfa6b --- /dev/null +++ b/platform/machine/external/flight.go @@ -0,0 +1,126 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package external + +import ( + "github.com/coreos/pkg/capnslog" + + "golang.org/x/crypto/ssh" + "golang.org/x/net/proxy" + + ctplatform "github.com/coreos/container-linux-config-transpiler/config/platform" + "github.com/coreos/mantle/network" + "github.com/coreos/mantle/platform" +) + +const ( + Platform platform.Name = "external" +) + +var ( + plog = capnslog.NewPackageLogger("github.com/coreos/mantle", "platform/machine/external") +) + +type flight struct { + *platform.BaseFlight + ManagementSSHClient *ssh.Client + ExternalOptions *Options +} + +type Options struct { + *platform.Options + ManagementHost string + ManagementUser string + ManagementPassword string + ManagementSocks string + // Executed on the Management Node + ProvisioningCmds string + SerialConsoleCmd string + DeprovisioningCmds string +} + +func newManagementSSHClient(opts *Options) (*ssh.Client, error) { + config := &ssh.ClientConfig{ + User: opts.ManagementUser, + Auth: []ssh.AuthMethod{ + ssh.Password(opts.ManagementPassword), + }, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + + if opts.ManagementSocks != "" { + dialer, err := proxy.SOCKS5("tcp", opts.ManagementSocks, nil, nil) + if err != nil { + return nil, err + } + conn, err := dialer.Dial("tcp", opts.ManagementHost) + if err != nil { + return nil, err + } + ncc, chans, reqs, err := ssh.NewClientConn(conn, opts.ManagementHost, config) + if err != nil { + return nil, err + } + return ssh.NewClient(ncc, chans, reqs), nil + } + client, err := ssh.Dial("tcp", opts.ManagementHost, config) + if err != nil { + return nil, err + } + return client, nil +} + +func NewFlight(opts *Options) (platform.Flight, error) { + managementSSHClient, err := newManagementSSHClient(opts) + if err != nil { + return nil, err + } + + retryDialer := network.RetryDialer{ + Dialer: managementSSHClient, + Retries: network.DefaultRetries, + } + bf, err := platform.NewBaseFlightWithDialer(opts.Options, Platform, ctplatform.Custom, &retryDialer) + if err != nil { + return nil, err + } + + pf := &flight{ + BaseFlight: bf, + ExternalOptions: opts, + ManagementSSHClient: managementSSHClient, + } + + return pf, nil +} + +func (pf *flight) NewCluster(rconf *platform.RuntimeConfig) (platform.Cluster, error) { + bc, err := platform.NewBaseCluster(pf.BaseFlight, rconf) + if err != nil { + return nil, err + } + + pc := &cluster{ + BaseCluster: bc, + flight: pf, + } + + pf.AddCluster(pc) + + return pc, nil +} + +func (pf *flight) Destroy() { + pf.BaseFlight.Destroy() + pf.ManagementSSHClient.Close() +} diff --git a/platform/machine/external/machine.go b/platform/machine/external/machine.go new file mode 100644 index 000000000..fb7d27fea --- /dev/null +++ b/platform/machine/external/machine.go @@ -0,0 +1,110 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package external + +import ( + "strings" + + "golang.org/x/crypto/ssh" + + "github.com/coreos/mantle/platform" +) + +type machine struct { + cluster *cluster + journal *platform.Journal + console *console + ipAddr string + rand string +} + +func (pm *machine) ID() string { + return pm.ipAddr + "_" + pm.rand +} + +func (pm *machine) IP() string { + return pm.ipAddr +} + +func (pm *machine) PrivateIP() string { + return pm.ipAddr +} + +func (pm *machine) RuntimeConf() platform.RuntimeConfig { + return pm.cluster.RuntimeConf() +} + +func (pm *machine) SSHClient() (*ssh.Client, error) { + return pm.cluster.SSHClient(pm.IP()) +} + +func (pm *machine) PasswordSSHClient(user string, password string) (*ssh.Client, error) { + return pm.cluster.PasswordSSHClient(pm.IP(), user, password) +} + +func (pm *machine) SSH(cmd string) ([]byte, []byte, error) { + return pm.cluster.SSH(pm, cmd) +} + +func (pm *machine) Reboot() error { + return platform.RebootMachine(pm, pm.journal) +} + +func (pm *machine) Destroy() { + if err := pm.cluster.deleteDevice(pm.ipAddr); err != nil { + plog.Errorf("Error terminating device %v: %v", pm.ID(), err) + } + + if pm.journal != nil { + pm.journal.Destroy() + } + + pm.cluster.DelMach(pm) +} + +func (pm *machine) ConsoleOutput() string { + if pm.console == nil { + return "" + } + output := pm.console.Output() + // The provisioning OS boots through iPXE and the real OS boots + // through GRUB. Try to ignore console logs from provisioning, but + // it's better to return everything than nothing. + grub := strings.Index(output, "GNU GRUB") + if grub == -1 { + plog.Warningf("Couldn't find GRUB banner in console output of %s", pm.ID()) + return output + } + linux := strings.Index(output[grub:], "Linux version") + if linux == -1 { + plog.Warningf("Couldn't find Linux banner in console output of %s", pm.ID()) + return output + } + return output[grub+linux:] +} + +func (pm *machine) JournalOutput() string { + if pm.journal == nil { + return "" + } + + data, err := pm.journal.Read() + if err != nil { + plog.Errorf("Reading journal for device %v: %v", pm.ID(), err) + } + return string(data) +} + +func (pm *machine) Board() string { + return pm.cluster.flight.Options().Board +} diff --git a/vendor/golang.org/x/net/internal/socks/client.go b/vendor/golang.org/x/net/internal/socks/client.go new file mode 100644 index 000000000..3d6f516a5 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socks/client.go @@ -0,0 +1,168 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package socks + +import ( + "context" + "errors" + "io" + "net" + "strconv" + "time" +) + +var ( + noDeadline = time.Time{} + aLongTimeAgo = time.Unix(1, 0) +) + +func (d *Dialer) connect(ctx context.Context, c net.Conn, address string) (_ net.Addr, ctxErr error) { + host, port, err := splitHostPort(address) + if err != nil { + return nil, err + } + if deadline, ok := ctx.Deadline(); ok && !deadline.IsZero() { + c.SetDeadline(deadline) + defer c.SetDeadline(noDeadline) + } + if ctx != context.Background() { + errCh := make(chan error, 1) + done := make(chan struct{}) + defer func() { + close(done) + if ctxErr == nil { + ctxErr = <-errCh + } + }() + go func() { + select { + case <-ctx.Done(): + c.SetDeadline(aLongTimeAgo) + errCh <- ctx.Err() + case <-done: + errCh <- nil + } + }() + } + + b := make([]byte, 0, 6+len(host)) // the size here is just an estimate + b = append(b, Version5) + if len(d.AuthMethods) == 0 || d.Authenticate == nil { + b = append(b, 1, byte(AuthMethodNotRequired)) + } else { + ams := d.AuthMethods + if len(ams) > 255 { + return nil, errors.New("too many authentication methods") + } + b = append(b, byte(len(ams))) + for _, am := range ams { + b = append(b, byte(am)) + } + } + if _, ctxErr = c.Write(b); ctxErr != nil { + return + } + + if _, ctxErr = io.ReadFull(c, b[:2]); ctxErr != nil { + return + } + if b[0] != Version5 { + return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0]))) + } + am := AuthMethod(b[1]) + if am == AuthMethodNoAcceptableMethods { + return nil, errors.New("no acceptable authentication methods") + } + if d.Authenticate != nil { + if ctxErr = d.Authenticate(ctx, c, am); ctxErr != nil { + return + } + } + + b = b[:0] + b = append(b, Version5, byte(d.cmd), 0) + if ip := net.ParseIP(host); ip != nil { + if ip4 := ip.To4(); ip4 != nil { + b = append(b, AddrTypeIPv4) + b = append(b, ip4...) + } else if ip6 := ip.To16(); ip6 != nil { + b = append(b, AddrTypeIPv6) + b = append(b, ip6...) + } else { + return nil, errors.New("unknown address type") + } + } else { + if len(host) > 255 { + return nil, errors.New("FQDN too long") + } + b = append(b, AddrTypeFQDN) + b = append(b, byte(len(host))) + b = append(b, host...) + } + b = append(b, byte(port>>8), byte(port)) + if _, ctxErr = c.Write(b); ctxErr != nil { + return + } + + if _, ctxErr = io.ReadFull(c, b[:4]); ctxErr != nil { + return + } + if b[0] != Version5 { + return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0]))) + } + if cmdErr := Reply(b[1]); cmdErr != StatusSucceeded { + return nil, errors.New("unknown error " + cmdErr.String()) + } + if b[2] != 0 { + return nil, errors.New("non-zero reserved field") + } + l := 2 + var a Addr + switch b[3] { + case AddrTypeIPv4: + l += net.IPv4len + a.IP = make(net.IP, net.IPv4len) + case AddrTypeIPv6: + l += net.IPv6len + a.IP = make(net.IP, net.IPv6len) + case AddrTypeFQDN: + if _, err := io.ReadFull(c, b[:1]); err != nil { + return nil, err + } + l += int(b[0]) + default: + return nil, errors.New("unknown address type " + strconv.Itoa(int(b[3]))) + } + if cap(b) < l { + b = make([]byte, l) + } else { + b = b[:l] + } + if _, ctxErr = io.ReadFull(c, b); ctxErr != nil { + return + } + if a.IP != nil { + copy(a.IP, b) + } else { + a.Name = string(b[:len(b)-2]) + } + a.Port = int(b[len(b)-2])<<8 | int(b[len(b)-1]) + return &a, nil +} + +func splitHostPort(address string) (string, int, error) { + host, port, err := net.SplitHostPort(address) + if err != nil { + return "", 0, err + } + portnum, err := strconv.Atoi(port) + if err != nil { + return "", 0, err + } + if 1 > portnum || portnum > 0xffff { + return "", 0, errors.New("port number out of range " + port) + } + return host, portnum, nil +} diff --git a/vendor/golang.org/x/net/internal/socks/socks.go b/vendor/golang.org/x/net/internal/socks/socks.go new file mode 100644 index 000000000..97db2340e --- /dev/null +++ b/vendor/golang.org/x/net/internal/socks/socks.go @@ -0,0 +1,317 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package socks provides a SOCKS version 5 client implementation. +// +// SOCKS protocol version 5 is defined in RFC 1928. +// Username/Password authentication for SOCKS version 5 is defined in +// RFC 1929. +package socks + +import ( + "context" + "errors" + "io" + "net" + "strconv" +) + +// A Command represents a SOCKS command. +type Command int + +func (cmd Command) String() string { + switch cmd { + case CmdConnect: + return "socks connect" + case cmdBind: + return "socks bind" + default: + return "socks " + strconv.Itoa(int(cmd)) + } +} + +// An AuthMethod represents a SOCKS authentication method. +type AuthMethod int + +// A Reply represents a SOCKS command reply code. +type Reply int + +func (code Reply) String() string { + switch code { + case StatusSucceeded: + return "succeeded" + case 0x01: + return "general SOCKS server failure" + case 0x02: + return "connection not allowed by ruleset" + case 0x03: + return "network unreachable" + case 0x04: + return "host unreachable" + case 0x05: + return "connection refused" + case 0x06: + return "TTL expired" + case 0x07: + return "command not supported" + case 0x08: + return "address type not supported" + default: + return "unknown code: " + strconv.Itoa(int(code)) + } +} + +// Wire protocol constants. +const ( + Version5 = 0x05 + + AddrTypeIPv4 = 0x01 + AddrTypeFQDN = 0x03 + AddrTypeIPv6 = 0x04 + + CmdConnect Command = 0x01 // establishes an active-open forward proxy connection + cmdBind Command = 0x02 // establishes a passive-open forward proxy connection + + AuthMethodNotRequired AuthMethod = 0x00 // no authentication required + AuthMethodUsernamePassword AuthMethod = 0x02 // use username/password + AuthMethodNoAcceptableMethods AuthMethod = 0xff // no acceptable authentication methods + + StatusSucceeded Reply = 0x00 +) + +// An Addr represents a SOCKS-specific address. +// Either Name or IP is used exclusively. +type Addr struct { + Name string // fully-qualified domain name + IP net.IP + Port int +} + +func (a *Addr) Network() string { return "socks" } + +func (a *Addr) String() string { + if a == nil { + return "" + } + port := strconv.Itoa(a.Port) + if a.IP == nil { + return net.JoinHostPort(a.Name, port) + } + return net.JoinHostPort(a.IP.String(), port) +} + +// A Conn represents a forward proxy connection. +type Conn struct { + net.Conn + + boundAddr net.Addr +} + +// BoundAddr returns the address assigned by the proxy server for +// connecting to the command target address from the proxy server. +func (c *Conn) BoundAddr() net.Addr { + if c == nil { + return nil + } + return c.boundAddr +} + +// A Dialer holds SOCKS-specific options. +type Dialer struct { + cmd Command // either CmdConnect or cmdBind + proxyNetwork string // network between a proxy server and a client + proxyAddress string // proxy server address + + // ProxyDial specifies the optional dial function for + // establishing the transport connection. + ProxyDial func(context.Context, string, string) (net.Conn, error) + + // AuthMethods specifies the list of request authentication + // methods. + // If empty, SOCKS client requests only AuthMethodNotRequired. + AuthMethods []AuthMethod + + // Authenticate specifies the optional authentication + // function. It must be non-nil when AuthMethods is not empty. + // It must return an error when the authentication is failed. + Authenticate func(context.Context, io.ReadWriter, AuthMethod) error +} + +// DialContext connects to the provided address on the provided +// network. +// +// The returned error value may be a net.OpError. When the Op field of +// net.OpError contains "socks", the Source field contains a proxy +// server address and the Addr field contains a command target +// address. +// +// See func Dial of the net package of standard library for a +// description of the network and address parameters. +func (d *Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + if err := d.validateTarget(network, address); err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + if ctx == nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")} + } + var err error + var c net.Conn + if d.ProxyDial != nil { + c, err = d.ProxyDial(ctx, d.proxyNetwork, d.proxyAddress) + } else { + var dd net.Dialer + c, err = dd.DialContext(ctx, d.proxyNetwork, d.proxyAddress) + } + if err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + a, err := d.connect(ctx, c, address) + if err != nil { + c.Close() + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + return &Conn{Conn: c, boundAddr: a}, nil +} + +// DialWithConn initiates a connection from SOCKS server to the target +// network and address using the connection c that is already +// connected to the SOCKS server. +// +// It returns the connection's local address assigned by the SOCKS +// server. +func (d *Dialer) DialWithConn(ctx context.Context, c net.Conn, network, address string) (net.Addr, error) { + if err := d.validateTarget(network, address); err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + if ctx == nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")} + } + a, err := d.connect(ctx, c, address) + if err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + return a, nil +} + +// Dial connects to the provided address on the provided network. +// +// Unlike DialContext, it returns a raw transport connection instead +// of a forward proxy connection. +// +// Deprecated: Use DialContext or DialWithConn instead. +func (d *Dialer) Dial(network, address string) (net.Conn, error) { + if err := d.validateTarget(network, address); err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + var err error + var c net.Conn + if d.ProxyDial != nil { + c, err = d.ProxyDial(context.Background(), d.proxyNetwork, d.proxyAddress) + } else { + c, err = net.Dial(d.proxyNetwork, d.proxyAddress) + } + if err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + if _, err := d.DialWithConn(context.Background(), c, network, address); err != nil { + c.Close() + return nil, err + } + return c, nil +} + +func (d *Dialer) validateTarget(network, address string) error { + switch network { + case "tcp", "tcp6", "tcp4": + default: + return errors.New("network not implemented") + } + switch d.cmd { + case CmdConnect, cmdBind: + default: + return errors.New("command not implemented") + } + return nil +} + +func (d *Dialer) pathAddrs(address string) (proxy, dst net.Addr, err error) { + for i, s := range []string{d.proxyAddress, address} { + host, port, err := splitHostPort(s) + if err != nil { + return nil, nil, err + } + a := &Addr{Port: port} + a.IP = net.ParseIP(host) + if a.IP == nil { + a.Name = host + } + if i == 0 { + proxy = a + } else { + dst = a + } + } + return +} + +// NewDialer returns a new Dialer that dials through the provided +// proxy server's network and address. +func NewDialer(network, address string) *Dialer { + return &Dialer{proxyNetwork: network, proxyAddress: address, cmd: CmdConnect} +} + +const ( + authUsernamePasswordVersion = 0x01 + authStatusSucceeded = 0x00 +) + +// UsernamePassword are the credentials for the username/password +// authentication method. +type UsernamePassword struct { + Username string + Password string +} + +// Authenticate authenticates a pair of username and password with the +// proxy server. +func (up *UsernamePassword) Authenticate(ctx context.Context, rw io.ReadWriter, auth AuthMethod) error { + switch auth { + case AuthMethodNotRequired: + return nil + case AuthMethodUsernamePassword: + if len(up.Username) == 0 || len(up.Username) > 255 || len(up.Password) == 0 || len(up.Password) > 255 { + return errors.New("invalid username/password") + } + b := []byte{authUsernamePasswordVersion} + b = append(b, byte(len(up.Username))) + b = append(b, up.Username...) + b = append(b, byte(len(up.Password))) + b = append(b, up.Password...) + // TODO(mikio): handle IO deadlines and cancelation if + // necessary + if _, err := rw.Write(b); err != nil { + return err + } + if _, err := io.ReadFull(rw, b[:2]); err != nil { + return err + } + if b[0] != authUsernamePasswordVersion { + return errors.New("invalid username/password version") + } + if b[1] != authStatusSucceeded { + return errors.New("username/password authentication failed") + } + return nil + } + return errors.New("unsupported authentication method " + strconv.Itoa(int(auth))) +} diff --git a/vendor/golang.org/x/net/proxy/dial.go b/vendor/golang.org/x/net/proxy/dial.go new file mode 100644 index 000000000..811c2e4e9 --- /dev/null +++ b/vendor/golang.org/x/net/proxy/dial.go @@ -0,0 +1,54 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proxy + +import ( + "context" + "net" +) + +// A ContextDialer dials using a context. +type ContextDialer interface { + DialContext(ctx context.Context, network, address string) (net.Conn, error) +} + +// Dial works like DialContext on net.Dialer but using a dialer returned by FromEnvironment. +// +// The passed ctx is only used for returning the Conn, not the lifetime of the Conn. +// +// Custom dialers (registered via RegisterDialerType) that do not implement ContextDialer +// can leak a goroutine for as long as it takes the underlying Dialer implementation to timeout. +// +// A Conn returned from a successful Dial after the context has been cancelled will be immediately closed. +func Dial(ctx context.Context, network, address string) (net.Conn, error) { + d := FromEnvironment() + if xd, ok := d.(ContextDialer); ok { + return xd.DialContext(ctx, network, address) + } + return dialContext(ctx, d, network, address) +} + +// WARNING: this can leak a goroutine for as long as the underlying Dialer implementation takes to timeout +// A Conn returned from a successful Dial after the context has been cancelled will be immediately closed. +func dialContext(ctx context.Context, d Dialer, network, address string) (net.Conn, error) { + var ( + conn net.Conn + done = make(chan struct{}, 1) + err error + ) + go func() { + conn, err = d.Dial(network, address) + close(done) + if conn != nil && ctx.Err() != nil { + conn.Close() + } + }() + select { + case <-ctx.Done(): + err = ctx.Err() + case <-done: + } + return conn, err +} diff --git a/vendor/golang.org/x/net/proxy/direct.go b/vendor/golang.org/x/net/proxy/direct.go new file mode 100644 index 000000000..3d66bdef9 --- /dev/null +++ b/vendor/golang.org/x/net/proxy/direct.go @@ -0,0 +1,31 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proxy + +import ( + "context" + "net" +) + +type direct struct{} + +// Direct implements Dialer by making network connections directly using net.Dial or net.DialContext. +var Direct = direct{} + +var ( + _ Dialer = Direct + _ ContextDialer = Direct +) + +// Dial directly invokes net.Dial with the supplied parameters. +func (direct) Dial(network, addr string) (net.Conn, error) { + return net.Dial(network, addr) +} + +// DialContext instantiates a net.Dialer and invokes its DialContext receiver with the supplied parameters. +func (direct) DialContext(ctx context.Context, network, addr string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, network, addr) +} diff --git a/vendor/golang.org/x/net/proxy/per_host.go b/vendor/golang.org/x/net/proxy/per_host.go new file mode 100644 index 000000000..573fe79e8 --- /dev/null +++ b/vendor/golang.org/x/net/proxy/per_host.go @@ -0,0 +1,155 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proxy + +import ( + "context" + "net" + "strings" +) + +// A PerHost directs connections to a default Dialer unless the host name +// requested matches one of a number of exceptions. +type PerHost struct { + def, bypass Dialer + + bypassNetworks []*net.IPNet + bypassIPs []net.IP + bypassZones []string + bypassHosts []string +} + +// NewPerHost returns a PerHost Dialer that directs connections to either +// defaultDialer or bypass, depending on whether the connection matches one of +// the configured rules. +func NewPerHost(defaultDialer, bypass Dialer) *PerHost { + return &PerHost{ + def: defaultDialer, + bypass: bypass, + } +} + +// Dial connects to the address addr on the given network through either +// defaultDialer or bypass. +func (p *PerHost) Dial(network, addr string) (c net.Conn, err error) { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + + return p.dialerForRequest(host).Dial(network, addr) +} + +// DialContext connects to the address addr on the given network through either +// defaultDialer or bypass. +func (p *PerHost) DialContext(ctx context.Context, network, addr string) (c net.Conn, err error) { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + d := p.dialerForRequest(host) + if x, ok := d.(ContextDialer); ok { + return x.DialContext(ctx, network, addr) + } + return dialContext(ctx, d, network, addr) +} + +func (p *PerHost) dialerForRequest(host string) Dialer { + if ip := net.ParseIP(host); ip != nil { + for _, net := range p.bypassNetworks { + if net.Contains(ip) { + return p.bypass + } + } + for _, bypassIP := range p.bypassIPs { + if bypassIP.Equal(ip) { + return p.bypass + } + } + return p.def + } + + for _, zone := range p.bypassZones { + if strings.HasSuffix(host, zone) { + return p.bypass + } + if host == zone[1:] { + // For a zone ".example.com", we match "example.com" + // too. + return p.bypass + } + } + for _, bypassHost := range p.bypassHosts { + if bypassHost == host { + return p.bypass + } + } + return p.def +} + +// AddFromString parses a string that contains comma-separated values +// specifying hosts that should use the bypass proxy. Each value is either an +// IP address, a CIDR range, a zone (*.example.com) or a host name +// (localhost). A best effort is made to parse the string and errors are +// ignored. +func (p *PerHost) AddFromString(s string) { + hosts := strings.Split(s, ",") + for _, host := range hosts { + host = strings.TrimSpace(host) + if len(host) == 0 { + continue + } + if strings.Contains(host, "/") { + // We assume that it's a CIDR address like 127.0.0.0/8 + if _, net, err := net.ParseCIDR(host); err == nil { + p.AddNetwork(net) + } + continue + } + if ip := net.ParseIP(host); ip != nil { + p.AddIP(ip) + continue + } + if strings.HasPrefix(host, "*.") { + p.AddZone(host[1:]) + continue + } + p.AddHost(host) + } +} + +// AddIP specifies an IP address that will use the bypass proxy. Note that +// this will only take effect if a literal IP address is dialed. A connection +// to a named host will never match an IP. +func (p *PerHost) AddIP(ip net.IP) { + p.bypassIPs = append(p.bypassIPs, ip) +} + +// AddNetwork specifies an IP range that will use the bypass proxy. Note that +// this will only take effect if a literal IP address is dialed. A connection +// to a named host will never match. +func (p *PerHost) AddNetwork(net *net.IPNet) { + p.bypassNetworks = append(p.bypassNetworks, net) +} + +// AddZone specifies a DNS suffix that will use the bypass proxy. A zone of +// "example.com" matches "example.com" and all of its subdomains. +func (p *PerHost) AddZone(zone string) { + if strings.HasSuffix(zone, ".") { + zone = zone[:len(zone)-1] + } + if !strings.HasPrefix(zone, ".") { + zone = "." + zone + } + p.bypassZones = append(p.bypassZones, zone) +} + +// AddHost specifies a host name that will use the bypass proxy. +func (p *PerHost) AddHost(host string) { + if strings.HasSuffix(host, ".") { + host = host[:len(host)-1] + } + p.bypassHosts = append(p.bypassHosts, host) +} diff --git a/vendor/golang.org/x/net/proxy/proxy.go b/vendor/golang.org/x/net/proxy/proxy.go new file mode 100644 index 000000000..9ff4b9a77 --- /dev/null +++ b/vendor/golang.org/x/net/proxy/proxy.go @@ -0,0 +1,149 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package proxy provides support for a variety of protocols to proxy network +// data. +package proxy // import "golang.org/x/net/proxy" + +import ( + "errors" + "net" + "net/url" + "os" + "sync" +) + +// A Dialer is a means to establish a connection. +// Custom dialers should also implement ContextDialer. +type Dialer interface { + // Dial connects to the given address via the proxy. + Dial(network, addr string) (c net.Conn, err error) +} + +// Auth contains authentication parameters that specific Dialers may require. +type Auth struct { + User, Password string +} + +// FromEnvironment returns the dialer specified by the proxy-related +// variables in the environment and makes underlying connections +// directly. +func FromEnvironment() Dialer { + return FromEnvironmentUsing(Direct) +} + +// FromEnvironmentUsing returns the dialer specify by the proxy-related +// variables in the environment and makes underlying connections +// using the provided forwarding Dialer (for instance, a *net.Dialer +// with desired configuration). +func FromEnvironmentUsing(forward Dialer) Dialer { + allProxy := allProxyEnv.Get() + if len(allProxy) == 0 { + return forward + } + + proxyURL, err := url.Parse(allProxy) + if err != nil { + return forward + } + proxy, err := FromURL(proxyURL, forward) + if err != nil { + return forward + } + + noProxy := noProxyEnv.Get() + if len(noProxy) == 0 { + return proxy + } + + perHost := NewPerHost(proxy, forward) + perHost.AddFromString(noProxy) + return perHost +} + +// proxySchemes is a map from URL schemes to a function that creates a Dialer +// from a URL with such a scheme. +var proxySchemes map[string]func(*url.URL, Dialer) (Dialer, error) + +// RegisterDialerType takes a URL scheme and a function to generate Dialers from +// a URL with that scheme and a forwarding Dialer. Registered schemes are used +// by FromURL. +func RegisterDialerType(scheme string, f func(*url.URL, Dialer) (Dialer, error)) { + if proxySchemes == nil { + proxySchemes = make(map[string]func(*url.URL, Dialer) (Dialer, error)) + } + proxySchemes[scheme] = f +} + +// FromURL returns a Dialer given a URL specification and an underlying +// Dialer for it to make network requests. +func FromURL(u *url.URL, forward Dialer) (Dialer, error) { + var auth *Auth + if u.User != nil { + auth = new(Auth) + auth.User = u.User.Username() + if p, ok := u.User.Password(); ok { + auth.Password = p + } + } + + switch u.Scheme { + case "socks5", "socks5h": + addr := u.Hostname() + port := u.Port() + if port == "" { + port = "1080" + } + return SOCKS5("tcp", net.JoinHostPort(addr, port), auth, forward) + } + + // If the scheme doesn't match any of the built-in schemes, see if it + // was registered by another package. + if proxySchemes != nil { + if f, ok := proxySchemes[u.Scheme]; ok { + return f(u, forward) + } + } + + return nil, errors.New("proxy: unknown scheme: " + u.Scheme) +} + +var ( + allProxyEnv = &envOnce{ + names: []string{"ALL_PROXY", "all_proxy"}, + } + noProxyEnv = &envOnce{ + names: []string{"NO_PROXY", "no_proxy"}, + } +) + +// envOnce looks up an environment variable (optionally by multiple +// names) once. It mitigates expensive lookups on some platforms +// (e.g. Windows). +// (Borrowed from net/http/transport.go) +type envOnce struct { + names []string + once sync.Once + val string +} + +func (e *envOnce) Get() string { + e.once.Do(e.init) + return e.val +} + +func (e *envOnce) init() { + for _, n := range e.names { + e.val = os.Getenv(n) + if e.val != "" { + return + } + } +} + +// reset is used by tests +func (e *envOnce) reset() { + e.once = sync.Once{} + e.val = "" +} diff --git a/vendor/golang.org/x/net/proxy/socks5.go b/vendor/golang.org/x/net/proxy/socks5.go new file mode 100644 index 000000000..c91651f96 --- /dev/null +++ b/vendor/golang.org/x/net/proxy/socks5.go @@ -0,0 +1,42 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proxy + +import ( + "context" + "net" + + "golang.org/x/net/internal/socks" +) + +// SOCKS5 returns a Dialer that makes SOCKSv5 connections to the given +// address with an optional username and password. +// See RFC 1928 and RFC 1929. +func SOCKS5(network, address string, auth *Auth, forward Dialer) (Dialer, error) { + d := socks.NewDialer(network, address) + if forward != nil { + if f, ok := forward.(ContextDialer); ok { + d.ProxyDial = func(ctx context.Context, network string, address string) (net.Conn, error) { + return f.DialContext(ctx, network, address) + } + } else { + d.ProxyDial = func(ctx context.Context, network string, address string) (net.Conn, error) { + return dialContext(ctx, forward, network, address) + } + } + } + if auth != nil { + up := socks.UsernamePassword{ + Username: auth.User, + Password: auth.Password, + } + d.AuthMethods = []socks.AuthMethod{ + socks.AuthMethodNotRequired, + socks.AuthMethodUsernamePassword, + } + d.Authenticate = up.Authenticate + } + return d, nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 3fed3befe..c9edce0db 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -399,7 +399,9 @@ golang.org/x/net/http/httpguts golang.org/x/net/http2 golang.org/x/net/http2/hpack golang.org/x/net/idna +golang.org/x/net/internal/socks golang.org/x/net/internal/timeseries +golang.org/x/net/proxy golang.org/x/net/trace # golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/oauth2