Move retry logic for scale from zero into rio

This moves the retry logic for the scale from zero functionality into
the Rio controller where it can be performed by the Envoy proxy (Gloo).

go.sum

@@ -139,6 +139,7 @@ github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb
 github.com/bluekeyes/hatpear v0.0.0-20180714193905-ffb42d5bb417/go.mod h1:D+WOahrNtu6OK0KiVoXY9h5j7IcEs5LYke+zJkMBsKg=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
+github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/bradleyfalzon/ghinstallation v0.1.2/go.mod h1:VQsLlCoNa54/CNXcc2DuCfNZrZxqQcyPeqKUugF/2h8=
 github.com/briandowns/spinner v0.0.0-20190212173954-5cf08d0ac778/go.mod h1:hw/JEQBIE+c/BLI4aKM8UU8v+ZqrD3h7HC27kKt8JQU=
@@ -827,8 +828,6 @@ github.com/radovskyb/watcher v1.0.7 h1:AYePLih6dpmS32vlHfhCeli8127LzkIgwJGcwwe8t
 github.com/radovskyb/watcher v1.0.7/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm+ZuvsUYIg=
 github.com/rancher/cert-manager v0.5.1-0.20191021233300-3a070253aeda h1:o8QdcpXJ/3PYThmGJw7PgQ8Kt0VS2Rxxe7SbqwSeOMs=
 github.com/rancher/cert-manager v0.5.1-0.20191021233300-3a070253aeda/go.mod h1:GisP3XX8Kr7jt/idFkYU62z6jTdEu6ZtUZvEiz7zSyw=
-github.com/rancher/gitwatcher v0.4.1 h1:zvRg2cWIwjQ63QVRRGSchRpI8yHFC/Up4bTrjI3qlm0=
-github.com/rancher/gitwatcher v0.4.1/go.mod h1:ewWLamldgMOOTm6ySgbekkbutk3xH1WkjKIOQ2L4p/s=
 github.com/rancher/gitwatcher v0.4.4 h1:jmgrp0cyLtuMWkfjYJVe+UEQ9ZxAxcXXtZFAu43MzHI=
 github.com/rancher/gitwatcher v0.4.4/go.mod h1:8dIsBtlsH0PrnBb9IoLbZaOEus/FnIGXm2QUAVpl2W8=
 github.com/rancher/kubernetes v1.16.2-k3s.1 h1:+oJEecXgQDkEOD/X8z2YUdYVonbXZtGzXsmtKDPYesg=
@@ -867,17 +866,12 @@ github.com/rancher/norman v0.0.0-20191114233102-966e8db9e670 h1:EM5VxDfLyKFTEPVV
 github.com/rancher/norman v0.0.0-20191114233102-966e8db9e670/go.mod h1:zOBq0bQmo9slllyM1kcKpKr1RH5IWodjySaffLKSOx0=
 github.com/rancher/rdns-server v0.5.7-0.20190927164127-7128efe7d065 h1:K2/79Jdw0viw+ryWSJQcIMYJf2vc5yQXlEHPTbpVPVs=
 github.com/rancher/rdns-server v0.5.7-0.20190927164127-7128efe7d065/go.mod h1:/yw1tSnAWDSDSmhKpYNA/FBA49fbbtNfh/IHT31v7jE=
-github.com/rancher/stern v0.0.0-20191203174401-30397523f082 h1:rDHE/uQXR2tjyik9kkVI8SN3eSliuqbjouC427M8VUM=
-github.com/rancher/stern v0.0.0-20191203174401-30397523f082/go.mod h1:aaP5eLMmFcyJUsZB6FwAO+0eILWeBKvupuHeeEuY0qQ=
 github.com/rancher/stern v0.0.0-20191213223518-59c2bf84f705 h1:LD7T78U5A4/tAZ+3jMRy6c6iJhhb+TARG29DdCtWF2Y=
 github.com/rancher/stern v0.0.0-20191213223518-59c2bf84f705/go.mod h1:aaP5eLMmFcyJUsZB6FwAO+0eILWeBKvupuHeeEuY0qQ=
-github.com/rancher/wrangler v0.1.4/go.mod h1:EYP7cqpg42YqElaCm+U9ieSrGQKAXxUH5xsr+XGpWyE=
-github.com/rancher/wrangler v0.1.7-0.20190824203417-e7b6ecb74e90/go.mod h1:TLc4vXF21FWz2MYNHny5zDwFwBqvgBViU5n9CHrJRQ0=
 github.com/rancher/wrangler v0.2.1-0.20191015042916-f2a6ecca4f20/go.mod h1:w9jivI6vlBZmqzizNQM0Li/ZwoNTBeDpIv+NtBoFiMI=
 github.com/rancher/wrangler v0.2.1-0.20191025041946-1fd360590735/go.mod h1:zmXTOSzU0vhumCyl0+Acq2FEP5WJOJRqnCQDegknyWA=
 github.com/rancher/wrangler v0.2.1-0.20191109002915-2a833f7e410d h1:lz9Aef0JEy2xmJhbU0jMfQ3Mh3pUFM9AP00pgUsQAlk=
 github.com/rancher/wrangler v0.2.1-0.20191109002915-2a833f7e410d/go.mod h1:zmXTOSzU0vhumCyl0+Acq2FEP5WJOJRqnCQDegknyWA=
-github.com/rancher/wrangler-api v0.1.4/go.mod h1:Mc8bNN5rNxpQ7am2orUbonahw4xJ6ztF+fUTSku8KnE=
 github.com/rancher/wrangler-api v0.2.1-0.20191015045805-d3635aa0853a/go.mod h1:Z6EJS8Z3aaptGHvr96E2FPTkRL6UgOURCR4YZbz8D5Q=
 github.com/rancher/wrangler-api v0.2.1-0.20191025043713-b1ca9c21825a h1:dH3D71jzKuYRY7qLe9AKuwz8DKO6G9PCfkeafETMmZo=
 github.com/rancher/wrangler-api v0.2.1-0.20191025043713-b1ca9c21825a/go.mod h1:U4nfgObQ7EERMwljGhCfQIbh2ql/t8suINX8/VfWiko=
@@ -900,8 +894,6 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
 github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
 github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
-github.com/sclevine/spec v1.3.0 h1:iTB51CYlnju5oRh0/l67fg1+RlQ2nqmFecwdvN+5TrI=
-github.com/sclevine/spec v1.3.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U=
 github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8=
 github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
@@ -1268,7 +1260,6 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
 istio.io/gogo-genproto v0.0.0-20190614210408-e88dc8b0e4db/go.mod h1:eIDJ6jNk/IeJz6ODSksHl5Aiczy5JUq6vFhJWI5OtiI=
 k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
-k8s.io/gengo v0.0.0-20190327210449-e17681d19d3a/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20190822140433-26a664648505 h1:ZY6yclUKVbZ+SdWnkfY+Je5vrMpKOxmGeKRbsXVmqYM=
 k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/heapster v1.2.0-beta.1/go.mod h1:h1uhptVXMwC8xtZBYsPXKVi8fpdlYkTs6k949KozGrM=

modules/gloo/pkg/vsfactory/vsfactory.go

@@ -2,6 +2,9 @@ package vsfactory
 
 import (
 	"net"
+	"time"
+
+	"github.com/solo-io/gloo/projects/gloo/pkg/api/v1/options/retries"
 
 	rioadminv1controller "github.com/rancher/rio/pkg/generated/controllers/admin.rio.cattle.io/v1"
 	"github.com/rancher/rio/types"
@@ -18,10 +21,16 @@ import (
 )
 
 const (
-	rioNameHeader      = "X-Rio-ServiceName"
-	rioNamespaceHeader = "X-Rio-Namespace"
+	rioNameHeader         = "X-Rio-ServiceName"
+	rioNamespaceHeader    = "X-Rio-Namespace"
+	scaleFromZeroAttempts = 10
 )
 
+// Retry configuration for the envoy gateway (Gloo)  https://www.envoyproxy.io/learn/automatic-retries
+// for services scaled to zero
+var overallTimeout = time.Second * 30
+var perTryTimeout = time.Second * 2
+
 type VirtualServiceFactory struct {
 	clusterDomainCache rioadminv1controller.ClusterDomainCache
 	publicDomainCache  rioadminv1controller.PublicDomainCache
@@ -145,8 +154,15 @@ func single(target target) *soloapiv1.Route_RouteAction {
 
 func newRoutePlugin(targets ...target) *v1.RouteOptions {
 	for _, t := range targets {
+		// use envoy to retry the connection if the service is being scaled up and increase envoy timeout
 		if t.ScaleIsZero {
 			return &v1.RouteOptions{
+				Timeout: &overallTimeout,
+				Retries: &retries.RetryPolicy{
+					RetryOn:       "5xx",
+					NumRetries:    scaleFromZeroAttempts,
+					PerTryTimeout: &perTryTimeout,
+				},
 				HeaderManipulation: &headers.HeaderManipulation{
 					RequestHeadersToAdd: []*headers.HeaderValueOption{
 						{

stacks/bindata.go

@@ -252,7 +252,7 @@ func stacksLinkerdInstallStackYaml() (*asset, error) {
 	return a, nil
 }
 
-var _stacksRioAutoscalerStackYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x84\x90\xc1\x6a\xf3\x30\x10\x84\xef\x7e\x0a\x1d\x7e\x08\xfc\xd4\x8e\x7b\x28\x04\x41\x0e\xa1\xe4\xd8\x12\xc8\x03\x94\xb5\xbc\x71\x05\x92\x56\x68\x57\x6a\x43\xe9\xbb\x17\xbb\x89\xe3\x42\xa1\x37\xcd\xce\xb7\xcc\x8e\x18\x53\xb1\x06\x59\x57\x4a\x41\x16\x62\x03\x0e\xd3\xa8\x94\x2a\x98\xd8\x52\xd0\xaa\xb4\x93\x1e\x1c\x75\xe0\x5e\x22\x26\x6f\x79\x74\xf8\x9b\xab\xd5\xea\xbf\x8a\xd4\xf3\xea\x26\x0d\x85\x93\x1d\x3c\xc4\xe5\x70\x0e\x68\x92\xa5\xc6\x80\x88\xc3\xc6\xd2\xfa\x7a\xc4\x94\x8d\x86\xbc\xc7\xd0\x83\x8c\x09\x8b\xed\xdf\x77\xfe\x04\xd6\x2c\x20\xf9\xc2\x59\x0f\x03\x6a\x95\x20\x98\x57\x4c\xeb\x64\xa9\x5e\xb4\x2e\x6d\x73\xdf\x3c\xdc\xc0\x43\x76\xee\x40\xce\x9a\xb3\x56\xe0\xde\xe0\xcc\x93\x87\xa1\x5c\x8b\x3f\xef\x9e\xf6\xc7\xc3\xee\x71\xbf\xfd\xf7\x31\xbf\x3f\x27\x33\x52\x92\xf9\x83\x36\xad\xde\xb4\x77\xf8\x1e\x89\x71\x7b\x02\xc7\x38\x19\x90\x86\x19\xf9\x79\xcb\x65\x58\xd7\x3d\x76\x79\xa8\x2a\x41\x1f\x1d\x08\x8e\x34\x86\x72\xcc\x1d\x8b\x56\x92\x32\x56\x5f\x01\x00\x00\xff\xff\xb9\x61\x17\x96\xc3\x01\x00\x00")
+var _stacksRioAutoscalerStackYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x84\x90\xc1\x6a\xf3\x30\x10\x84\xef\x7e\x0a\x1d\x7e\x08\xfc\xd4\x8e\x7b\x29\x41\x90\x43\x28\x39\xb6\x04\xf2\x00\x65\x2d\x6f\x5c\x81\xa4\x15\xda\x95\xda\x50\xfa\xee\xc5\x6e\xe2\xb8\x50\xe8\x4d\xb3\xf3\x2d\xb3\x23\xc6\x54\xac\x41\xd6\x95\x52\x90\x85\xd8\x80\xc3\x34\x2a\xa5\x0a\x26\xb6\x14\xb4\x2a\xed\xa4\x07\x47\x1d\xb8\x97\x88\xc9\x5b\x1e\x1d\xfe\xe6\x6a\xb5\xfa\xaf\x22\xf5\xbc\xba\x49\x43\xe1\x64\x07\x0f\x71\x39\x9c\x03\x9a\x64\xa9\x31\x20\xe2\xb0\xb1\xb4\xbe\x1e\x31\x65\xa3\x21\xef\x31\xf4\x20\x63\xc2\x62\xfb\xf7\x9d\x3f\x81\x35\x0b\x48\xbe\x70\xd6\xc3\x80\x5a\x25\x08\xe6\x15\xd3\x3a\x59\xaa\x17\xad\x4b\xdb\xdc\x37\x0f\x37\xf0\x90\x9d\x3b\x90\xb3\xe6\xac\x15\xb8\x37\x38\xf3\xe4\x61\x28\xd7\xe2\xcf\xbb\xa7\xfd\xf1\xb0\x7b\xdc\x6f\xff\x7d\xcc\xef\xcf\xc9\x8c\x94\x64\xfe\xa0\x4d\xab\x37\xed\x1d\xbe\x47\x62\xdc\x9e\xc0\x31\x4e\x06\xa4\x61\x46\x7e\xde\x72\x19\xd6\x75\x8f\x5d\x1e\xaa\x4a\xd0\x47\x07\x82\x23\x8d\xa1\x1c\x73\xc7\xa2\x95\xa4\x8c\xd5\x57\x00\x00\x00\xff\xff\xb2\x3a\x97\xa8\xc3\x01\x00\x00")
 
 func stacksRioAutoscalerStackYamlBytes() ([]byte, error) {
 	return bindataRead(

stacks/rio-autoscaler-stack.yaml

@@ -7,7 +7,7 @@ services:
     - '* autoscale.rio.cattle.io/servicescalerecommendations'
     - '* rio.cattle.io/services'
     - '* rio.cattle.io/services/status'
-    image: rancher/rio-autoscaler:v0.1.5
+    image: rancher/rio-autoscaler:v0.1.6
     imagePullPolicy: always
     env:
     - NAMESPACE=${NAMESPACE}