Alex/fix tests

Makefile

@@ -47,6 +47,8 @@ production: clean
 #
 test: 
 	go test -v github.com/gravitational/teleport/lib/... -cover
+	go test -v github.com/gravitational/teleport/tool/tsh -cover
+
 
 test-with-etcd: install
 	${ETCD_FLAGS} go test -v -test.parallel=0 $(shell go list ./... | grep -v /vendor/) -cover

lib/auth/clt_retries.go

@@ -0,0 +1,67 @@
+/*
+Copyright 2015 Gravitational, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package auth
+
+import (
+	"github.com/gravitational/teleport/lib/services"
+
+	"github.com/gravitational/trace"
+)
+
+type limitedClient interface {
+	GetServers() ([]services.Server, error)
+	GetCertAuthorities(caType services.CertAuthType) ([]*services.CertAuthority, error)
+}
+
+type retryingClient struct {
+	limitedClient
+	retries int
+}
+
+func RetryingClient(client limitedClient, retries int) *retryingClient {
+	if retries < 1 {
+		retries = 1
+	}
+	return &retryingClient{
+		limitedClient: client,
+		retries:       retries,
+	}
+}
+
+func (c *retryingClient) GetServers() ([]services.Server, error) {
+	var e error
+	for i := 0; i < c.retries; i++ {
+		servers, err := c.limitedClient.GetServers()
+		if err == nil {
+			return servers, nil
+		}
+		e = err
+	}
+	return nil, trace.Wrap(e)
+}
+
+func (c *retryingClient) GetCertAuthorities(caType services.CertAuthType) ([]*services.CertAuthority, error) {
+	var e error
+	for i := 0; i < c.retries; i++ {
+		cas, err := c.limitedClient.GetCertAuthorities(caType)
+		if err == nil {
+			return cas, nil
+		}
+		e = err
+	}
+	return nil, trace.Wrap(e)
+}

lib/client/client_test.go

@@ -73,6 +73,7 @@ type ClientSuite struct {
 	otp          *hotp.HOTP
 	user         string
 	pass         []byte
+	freePorts    []string
 }
 
 var _ = Suite(&ClientSuite{})
@@ -84,6 +85,10 @@ func (s *ClientSuite) SetUpSuite(c *C) {
 	s.dir2 = c.MkDir()
 
 	allowAllLimiter, err := limiter.NewLimiter(limiter.LimiterConfig{})
+	c.Assert(err, IsNil)
+
+	s.freePorts, err = utils.GetFreeTCPPorts(10)
+	c.Assert(err, IsNil)
 
 	baseBk, err := boltbk.New(filepath.Join(s.dir, "db"))
 	c.Assert(err, IsNil)
@@ -123,7 +128,8 @@ func (s *ClientSuite) SetUpSuite(c *C) {
 		nil)
 
 	// Starting node1
-	s.srvAddress = "127.0.0.1:30187"
+	s.srvAddress = "127.0.0.1:" + s.freePorts[len(s.freePorts)-1]
+	s.freePorts = s.freePorts[:len(s.freePorts)-1]
 	s.srv, err = srv.New(
 		utils.NetAddr{AddrNetwork: "tcp", Addr: s.srvAddress},
 		"alice",
@@ -145,7 +151,8 @@ func (s *ClientSuite) SetUpSuite(c *C) {
 	c.Assert(s.srv.Start(), IsNil)
 
 	// Starting node2
-	s.srv2Address = "127.0.0.1:30189"
+	s.srv2Address = "127.0.0.1:" + s.freePorts[len(s.freePorts)-1]
+	s.freePorts = s.freePorts[:len(s.freePorts)-1]
 	s.srv2, err = srv.New(
 		utils.NetAddr{AddrNetwork: "tcp", Addr: s.srv2Address},
 		"bob",
@@ -172,16 +179,19 @@ func (s *ClientSuite) SetUpSuite(c *C) {
 	c.Assert(s.srv2.Start(), IsNil)
 
 	// Starting proxy
-	reverseTunnelAddress := utils.NetAddr{AddrNetwork: "tcp", Addr: "localhost:33057"}
+	reverseTunnelPort := s.freePorts[len(s.freePorts)-1]
+	s.freePorts = s.freePorts[:len(s.freePorts)-1]
+	c.Assert(err, IsNil)
+	reverseTunnelAddress := utils.NetAddr{AddrNetwork: "tcp", Addr: "localhost:" + reverseTunnelPort}
 	reverseTunnelServer, err := reversetunnel.NewServer(
 		reverseTunnelAddress,
 		[]ssh.Signer{s.signer},
 		s.roleAuth, allowAllLimiter)
 	c.Assert(err, IsNil)
 	c.Assert(reverseTunnelServer.Start(), IsNil)
 
-	s.proxyAddress = "localhost:34783"
-
+	s.proxyAddress = "localhost:" + s.freePorts[len(s.freePorts)-1]
+	s.freePorts = s.freePorts[:len(s.freePorts)-1]
 	s.proxy, err = srv.New(
 		utils.NetAddr{AddrNetwork: "tcp", Addr: s.proxyAddress},
 		"localhost",
@@ -201,7 +211,7 @@ func (s *ClientSuite) SetUpSuite(c *C) {
 	go apiSrv.Serve()
 
 	tsrv, err := auth.NewTunServer(
-		utils.NetAddr{AddrNetwork: "tcp", Addr: "localhost:31498"},
+		utils.NetAddr{AddrNetwork: "tcp", Addr: "localhost:0"},
 		[]ssh.Signer{s.signer},
 		apiSrv, s.a, allowAllLimiter)
 	c.Assert(err, IsNil)
@@ -479,7 +489,7 @@ func (s *ClientSuite) TestGetServer(c *C) {
 	c.Assert(err, IsNil)
 
 	server1Info := services.Server{
-		ID:       "alice",
+		ID:       s.srvAddress,
 		Addr:     s.srvAddress,
 		Hostname: "alice",
 		Labels: map[string]string{
@@ -496,7 +506,7 @@ func (s *ClientSuite) TestGetServer(c *C) {
 	}
 
 	server2Info := services.Server{
-		ID:       "bob",
+		ID:       s.srv2Address,
 		Addr:     s.srv2Address,
 		Hostname: "bob",
 		Labels: map[string]string{
@@ -518,24 +528,45 @@ func (s *ClientSuite) TestGetServer(c *C) {
 
 	servers, err := proxyClient.GetServers()
 	c.Assert(err, IsNil)
-	c.Assert(servers, DeepEquals, []services.Server{
-		server1Info,
-		server2Info,
-	})
+	if servers[0].Addr == server1Info.Addr {
+		c.Assert(servers, DeepEquals, []services.Server{
+			server1Info,
+			server2Info,
+		})
+	} else {
+		c.Assert(servers, DeepEquals, []services.Server{
+			server2Info,
+			server1Info,
+		})
+	}
 
 	servers, err = proxyClient.FindServers("label1", "value1")
 	c.Assert(err, IsNil)
-	c.Assert(servers, DeepEquals, []services.Server{
-		server1Info,
-		server2Info,
-	})
+	if servers[0].Addr == server1Info.Addr {
+		c.Assert(servers, DeepEquals, []services.Server{
+			server1Info,
+			server2Info,
+		})
+	} else {
+		c.Assert(servers, DeepEquals, []services.Server{
+			server2Info,
+			server1Info,
+		})
+	}
 
 	servers, err = proxyClient.FindServers("label1", "val.*")
 	c.Assert(err, IsNil)
-	c.Assert(servers, DeepEquals, []services.Server{
-		server1Info,
-		server2Info,
-	})
+	if servers[0].Addr == server1Info.Addr {
+		c.Assert(servers, DeepEquals, []services.Server{
+			server1Info,
+			server2Info,
+		})
+	} else {
+		c.Assert(servers, DeepEquals, []services.Server{
+			server2Info,
+			server1Info,
+		})
+	}
 
 	servers, err = proxyClient.FindServers("label2", ".*ue2")
 	c.Assert(err, IsNil)

lib/hangout/hangout_test.go

@@ -265,52 +265,59 @@ func (s *HangoutsSuite) TestHangout(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(out, Equals, " expr 11 + 22\r\n33\r\n$")
 
-	// Initializing tsh join
-	proxy, err := client.ConnectToProxy(s.proxyAddress,
-		[]ssh.AuthMethod{s.teleagent.AuthMethod()}, client.CheckHostSignerFromCache, "anyuser")
-	c.Assert(err, IsNil)
+	for i := 0; i < 3; i++ {
 
-	authConn, err := proxy.ConnectToHangout(hangoutServer.HangoutID+":"+utils.HangoutAuthPortAlias, []ssh.AuthMethod{s.teleagent.AuthMethod()})
-	c.Assert(err, IsNil)
+		// Initializing tsh join
+		proxy, err := client.ConnectToProxy(s.proxyAddress,
+			[]ssh.AuthMethod{s.teleagent.AuthMethod()}, client.CheckHostSignerFromCache, "anyuser")
+		c.Assert(err, IsNil)
 
-	authClient, err := auth.NewClientFromSSHClient(authConn.Client)
-	c.Assert(err, IsNil)
+		authConn, err := proxy.ConnectToHangout(hangoutServer.HangoutID+":"+utils.HangoutAuthPortAlias, []ssh.AuthMethod{s.teleagent.AuthMethod()})
+		c.Assert(err, IsNil)
 
-	nodeAuthMethod, err := Authorize(authClient)
-	c.Assert(err, IsNil)
+		authClient, err := auth.NewClientFromSSHClient(authConn.Client)
+		c.Assert(err, IsNil)
 
-	nodeConn, err := proxy.ConnectToHangout(hangoutServer.HangoutID+":"+utils.HangoutNodePortAlias, []ssh.AuthMethod{nodeAuthMethod})
-	c.Assert(err, IsNil)
+		nodeAuthMethod, err := Authorize(authClient)
+		c.Assert(err, IsNil)
 
-	shell2, err := nodeConn.Shell(100, 100, "hangoutSession")
-	c.Assert(err, IsNil)
-	shell2Reader := bufio.NewReader(shell2)
-	// tsh join initialized
+		c.Assert(authConn.Close(), IsNil)
 
-	// run second command
-	_, err = shell1.Write([]byte("expr 2 + 3\n"))
-	c.Assert(err, IsNil)
+		nodeConn, err := proxy.ConnectToHangout(hangoutServer.HangoutID+":"+utils.HangoutNodePortAlias, []ssh.AuthMethod{nodeAuthMethod})
+		c.Assert(err, IsNil)
 
-	out, err = shell1Reader.ReadString('$')
-	c.Assert(err, IsNil)
-	c.Assert(out, Equals, " expr 2 + 3\r\n5\r\n$")
+		shell2, err := nodeConn.Shell(100, 100, "hangoutSession")
+		c.Assert(err, IsNil)
+		shell2Reader := bufio.NewReader(shell2)
+		// tsh join initialized
 
-	out, err = shell2Reader.ReadString('$')
-	c.Assert(err, IsNil)
-	c.Assert(out, Equals, "expr 2 + 3\r\n5\r\n$")
+		// run second command
+		_, err = shell1.Write([]byte("expr 2 + 3\n"))
+		c.Assert(err, IsNil)
 
-	// run third command
-	_, err = shell2.Write([]byte("expr 6 + 2\n"))
-	c.Assert(err, IsNil)
+		out, err = shell1Reader.ReadString('$')
+		c.Assert(err, IsNil)
+		c.Assert(out, Equals, " expr 2 + 3\r\n5\r\n$")
 
-	out, err = shell1Reader.ReadString('$')
-	c.Assert(err, IsNil)
-	c.Assert(out, Equals, " expr 6 + 2\r\n8\r\n$")
+		out, err = shell2Reader.ReadString('$')
+		c.Assert(err, IsNil)
+		c.Assert(out, Equals, "expr 2 + 3\r\n5\r\n$")
 
-	out, err = shell2Reader.ReadString('$')
-	c.Assert(err, IsNil)
-	c.Assert(out, Equals, " expr 6 + 2\r\n8\r\n$")
+		// run third command
+		_, err = shell2.Write([]byte("expr 6 + 2\n"))
+		c.Assert(err, IsNil)
+
+		out, err = shell1Reader.ReadString('$')
+		c.Assert(err, IsNil)
+		c.Assert(out, Equals, " expr 6 + 2\r\n8\r\n$")
 
+		out, err = shell2Reader.ReadString('$')
+		c.Assert(err, IsNil)
+		c.Assert(out, Equals, " expr 6 + 2\r\n8\r\n$")
+
+		c.Assert(shell2.Close(), IsNil)
+		c.Assert(nodeConn.Close(), IsNil)
+		c.Assert(proxy.Close(), IsNil)
+	}
 	c.Assert(shell1.Close(), IsNil)
-	c.Assert(shell2.Close(), IsNil)
 }

lib/reversetunnel/srv.go

@@ -587,6 +587,7 @@ func newRemoteSite(srv *server, domainName string) (*tunnelSite, error) {
 		"http://stub:0",
 		roundtrip.HTTPClient(&http.Client{
 			Transport: remoteSite.transport,
+			Timeout:   5 * time.Second,
 		}))
 	if err != nil {
 		return nil, trace.Wrap(err)

lib/srv/hangouts.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/gravitational/teleport/lib/auth"
 	"github.com/gravitational/teleport/lib/services"
 	"github.com/gravitational/teleport/lib/utils"
 
@@ -72,7 +73,7 @@ func (t *hangoutsSubsys) execute(sconn *ssh.ServerConn, ch ssh.Channel, req *ssh
 		return trace.Wrap(err)
 	}
 
-	servers, err := clt.GetServers()
+	servers, err := auth.RetryingClient(clt, 20).GetServers()
 	if err != nil {
 		return trace.Wrap(err)
 	}

lib/srv/proxy.go

@@ -58,10 +58,12 @@ func (t *proxySubsys) execute(sconn *ssh.ServerConn, ch ssh.Channel, req *ssh.Re
 	serverAddr := fmt.Sprintf("%v:%v", t.host, t.port)
 	var server *services.Server
 	for i := range servers {
+
 		ip, port, err := net.SplitHostPort(servers[i].Addr)
 		if err != nil {
 			return trace.Wrap(err)
 		}
+
 		// match either by hostname of ip, based on the match
 		if (t.host == ip || t.host == servers[i].Hostname) && port == t.port {
 			server = &servers[i]

lib/srv/srv.go

@@ -181,7 +181,7 @@ func (s *Server) Addr() string {
 }
 
 func (s *Server) ID() string {
-	return s.hostname
+	return s.addr.Addr
 }
 
 func (s *Server) heartbeatPresence() {
@@ -299,7 +299,7 @@ func (s *Server) checkPermissionToLogin(cert ssh.PublicKey, teleportUser, osUser
 // key is the real CA authority key.
 func (s *Server) isAuthority(cert ssh.PublicKey) bool {
 	// find cert authority by it's key
-	cas, err := s.ap.GetCertAuthorities(services.UserCA)
+	cas, err := auth.RetryingClient(s.ap, 20).GetCertAuthorities(services.UserCA)
 	if err != nil {
 		log.Warningf("%v", err)
 		return false