Skip to content
This repository has been archived by the owner on Sep 5, 2024. It is now read-only.

Move retry logic for scale from zero into rio #952

Merged
merged 1 commit into from
Dec 23, 2019
Merged

Move retry logic for scale from zero into rio #952

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 1 addition & 10 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb
github.com/bluekeyes/hatpear v0.0.0-20180714193905-ffb42d5bb417/go.mod h1:D+WOahrNtu6OK0KiVoXY9h5j7IcEs5LYke+zJkMBsKg=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
github.com/bradleyfalzon/ghinstallation v0.1.2/go.mod h1:VQsLlCoNa54/CNXcc2DuCfNZrZxqQcyPeqKUugF/2h8=
github.com/briandowns/spinner v0.0.0-20190212173954-5cf08d0ac778/go.mod h1:hw/JEQBIE+c/BLI4aKM8UU8v+ZqrD3h7HC27kKt8JQU=
Expand Down Expand Up @@ -827,8 +828,6 @@ github.com/radovskyb/watcher v1.0.7 h1:AYePLih6dpmS32vlHfhCeli8127LzkIgwJGcwwe8t
github.com/radovskyb/watcher v1.0.7/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm+ZuvsUYIg=
github.com/rancher/cert-manager v0.5.1-0.20191021233300-3a070253aeda h1:o8QdcpXJ/3PYThmGJw7PgQ8Kt0VS2Rxxe7SbqwSeOMs=
github.com/rancher/cert-manager v0.5.1-0.20191021233300-3a070253aeda/go.mod h1:GisP3XX8Kr7jt/idFkYU62z6jTdEu6ZtUZvEiz7zSyw=
github.com/rancher/gitwatcher v0.4.1 h1:zvRg2cWIwjQ63QVRRGSchRpI8yHFC/Up4bTrjI3qlm0=
github.com/rancher/gitwatcher v0.4.1/go.mod h1:ewWLamldgMOOTm6ySgbekkbutk3xH1WkjKIOQ2L4p/s=
github.com/rancher/gitwatcher v0.4.4 h1:jmgrp0cyLtuMWkfjYJVe+UEQ9ZxAxcXXtZFAu43MzHI=
github.com/rancher/gitwatcher v0.4.4/go.mod h1:8dIsBtlsH0PrnBb9IoLbZaOEus/FnIGXm2QUAVpl2W8=
github.com/rancher/kubernetes v1.16.2-k3s.1 h1:+oJEecXgQDkEOD/X8z2YUdYVonbXZtGzXsmtKDPYesg=
Expand Down Expand Up @@ -867,17 +866,12 @@ github.com/rancher/norman v0.0.0-20191114233102-966e8db9e670 h1:EM5VxDfLyKFTEPVV
github.com/rancher/norman v0.0.0-20191114233102-966e8db9e670/go.mod h1:zOBq0bQmo9slllyM1kcKpKr1RH5IWodjySaffLKSOx0=
github.com/rancher/rdns-server v0.5.7-0.20190927164127-7128efe7d065 h1:K2/79Jdw0viw+ryWSJQcIMYJf2vc5yQXlEHPTbpVPVs=
github.com/rancher/rdns-server v0.5.7-0.20190927164127-7128efe7d065/go.mod h1:/yw1tSnAWDSDSmhKpYNA/FBA49fbbtNfh/IHT31v7jE=
github.com/rancher/stern v0.0.0-20191203174401-30397523f082 h1:rDHE/uQXR2tjyik9kkVI8SN3eSliuqbjouC427M8VUM=
github.com/rancher/stern v0.0.0-20191203174401-30397523f082/go.mod h1:aaP5eLMmFcyJUsZB6FwAO+0eILWeBKvupuHeeEuY0qQ=
github.com/rancher/stern v0.0.0-20191213223518-59c2bf84f705 h1:LD7T78U5A4/tAZ+3jMRy6c6iJhhb+TARG29DdCtWF2Y=
github.com/rancher/stern v0.0.0-20191213223518-59c2bf84f705/go.mod h1:aaP5eLMmFcyJUsZB6FwAO+0eILWeBKvupuHeeEuY0qQ=
github.com/rancher/wrangler v0.1.4/go.mod h1:EYP7cqpg42YqElaCm+U9ieSrGQKAXxUH5xsr+XGpWyE=
github.com/rancher/wrangler v0.1.7-0.20190824203417-e7b6ecb74e90/go.mod h1:TLc4vXF21FWz2MYNHny5zDwFwBqvgBViU5n9CHrJRQ0=
github.com/rancher/wrangler v0.2.1-0.20191015042916-f2a6ecca4f20/go.mod h1:w9jivI6vlBZmqzizNQM0Li/ZwoNTBeDpIv+NtBoFiMI=
github.com/rancher/wrangler v0.2.1-0.20191025041946-1fd360590735/go.mod h1:zmXTOSzU0vhumCyl0+Acq2FEP5WJOJRqnCQDegknyWA=
github.com/rancher/wrangler v0.2.1-0.20191109002915-2a833f7e410d h1:lz9Aef0JEy2xmJhbU0jMfQ3Mh3pUFM9AP00pgUsQAlk=
github.com/rancher/wrangler v0.2.1-0.20191109002915-2a833f7e410d/go.mod h1:zmXTOSzU0vhumCyl0+Acq2FEP5WJOJRqnCQDegknyWA=
github.com/rancher/wrangler-api v0.1.4/go.mod h1:Mc8bNN5rNxpQ7am2orUbonahw4xJ6ztF+fUTSku8KnE=
github.com/rancher/wrangler-api v0.2.1-0.20191015045805-d3635aa0853a/go.mod h1:Z6EJS8Z3aaptGHvr96E2FPTkRL6UgOURCR4YZbz8D5Q=
github.com/rancher/wrangler-api v0.2.1-0.20191025043713-b1ca9c21825a h1:dH3D71jzKuYRY7qLe9AKuwz8DKO6G9PCfkeafETMmZo=
github.com/rancher/wrangler-api v0.2.1-0.20191025043713-b1ca9c21825a/go.mod h1:U4nfgObQ7EERMwljGhCfQIbh2ql/t8suINX8/VfWiko=
Expand All @@ -900,8 +894,6 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sclevine/spec v1.3.0 h1:iTB51CYlnju5oRh0/l67fg1+RlQ2nqmFecwdvN+5TrI=
github.com/sclevine/spec v1.3.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U=
github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8=
github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
Expand Down Expand Up @@ -1268,7 +1260,6 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
istio.io/gogo-genproto v0.0.0-20190614210408-e88dc8b0e4db/go.mod h1:eIDJ6jNk/IeJz6ODSksHl5Aiczy5JUq6vFhJWI5OtiI=
k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/gengo v0.0.0-20190327210449-e17681d19d3a/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/gengo v0.0.0-20190822140433-26a664648505 h1:ZY6yclUKVbZ+SdWnkfY+Je5vrMpKOxmGeKRbsXVmqYM=
k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/heapster v1.2.0-beta.1/go.mod h1:h1uhptVXMwC8xtZBYsPXKVi8fpdlYkTs6k949KozGrM=
Expand Down
20 changes: 18 additions & 2 deletions modules/gloo/pkg/vsfactory/vsfactory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ package vsfactory

import (
"net"
"time"

"github.com/solo-io/gloo/projects/gloo/pkg/api/v1/options/retries"

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: space

rioadminv1controller "github.com/rancher/rio/pkg/generated/controllers/admin.rio.cattle.io/v1"
"github.com/rancher/rio/types"
Expand All @@ -18,10 +21,16 @@ import (
)

const (
rioNameHeader = "X-Rio-ServiceName"
rioNamespaceHeader = "X-Rio-Namespace"
rioNameHeader = "X-Rio-ServiceName"
rioNamespaceHeader = "X-Rio-Namespace"
scaleFromZeroAttempts = 10
)

// Retry configuration for the envoy gateway (Gloo) https://www.envoyproxy.io/learn/automatic-retries
// for services scaled to zero
var overallTimeout = time.Second * 30
var perTryTimeout = time.Second * 2
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was what I saw as a decent timeout for each request. Ultimately I think this may need to be configurable based upon the service being scaled


type VirtualServiceFactory struct {
clusterDomainCache rioadminv1controller.ClusterDomainCache
publicDomainCache rioadminv1controller.PublicDomainCache
Expand Down Expand Up @@ -145,8 +154,15 @@ func single(target target) *soloapiv1.Route_RouteAction {

func newRoutePlugin(targets ...target) *v1.RouteOptions {
for _, t := range targets {
// use envoy to retry the connection if the service is being scaled up and increase envoy timeout
if t.ScaleIsZero {
return &v1.RouteOptions{
Timeout: &overallTimeout,
Retries: &retries.RetryPolicy{
RetryOn: "5xx",
NumRetries: scaleFromZeroAttempts,
PerTryTimeout: &perTryTimeout,
},
HeaderManipulation: &headers.HeaderManipulation{
RequestHeadersToAdd: []*headers.HeaderValueOption{
{
Expand Down
2 changes: 1 addition & 1 deletion stacks/bindata.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ func stacksLinkerdInstallStackYaml() (*asset, error) {
return a, nil
}

var _stacksRioAutoscalerStackYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x84\x90\xc1\x6a\xf3\x30\x10\x84\xef\x7e\x0a\x1d\x7e\x08\xfc\xd4\x8e\x7b\x28\x04\x41\x0e\xa1\xe4\xd8\x12\xc8\x03\x94\xb5\xbc\x71\x05\x92\x56\x68\x57\x6a\x43\xe9\xbb\x17\xbb\x89\xe3\x42\xa1\x37\xcd\xce\xb7\xcc\x8e\x18\x53\xb1\x06\x59\x57\x4a\x41\x16\x62\x03\x0e\xd3\xa8\x94\x2a\x98\xd8\x52\xd0\xaa\xb4\x93\x1e\x1c\x75\xe0\x5e\x22\x26\x6f\x79\x74\xf8\x9b\xab\xd5\xea\xbf\x8a\xd4\xf3\xea\x26\x0d\x85\x93\x1d\x3c\xc4\xe5\x70\x0e\x68\x92\xa5\xc6\x80\x88\xc3\xc6\xd2\xfa\x7a\xc4\x94\x8d\x86\xbc\xc7\xd0\x83\x8c\x09\x8b\xed\xdf\x77\xfe\x04\xd6\x2c\x20\xf9\xc2\x59\x0f\x03\x6a\x95\x20\x98\x57\x4c\xeb\x64\xa9\x5e\xb4\x2e\x6d\x73\xdf\x3c\xdc\xc0\x43\x76\xee\x40\xce\x9a\xb3\x56\xe0\xde\xe0\xcc\x93\x87\xa1\x5c\x8b\x3f\xef\x9e\xf6\xc7\xc3\xee\x71\xbf\xfd\xf7\x31\xbf\x3f\x27\x33\x52\x92\xf9\x83\x36\xad\xde\xb4\x77\xf8\x1e\x89\x71\x7b\x02\xc7\x38\x19\x90\x86\x19\xf9\x79\xcb\x65\x58\xd7\x3d\x76\x79\xa8\x2a\x41\x1f\x1d\x08\x8e\x34\x86\x72\xcc\x1d\x8b\x56\x92\x32\x56\x5f\x01\x00\x00\xff\xff\xb9\x61\x17\x96\xc3\x01\x00\x00")
var _stacksRioAutoscalerStackYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x84\x90\xc1\x6a\xf3\x30\x10\x84\xef\x7e\x0a\x1d\x7e\x08\xfc\xd4\x8e\x7b\x29\x41\x90\x43\x28\x39\xb6\x04\xf2\x00\x65\x2d\x6f\x5c\x81\xa4\x15\xda\x95\xda\x50\xfa\xee\xc5\x6e\xe2\xb8\x50\xe8\x4d\xb3\xf3\x2d\xb3\x23\xc6\x54\xac\x41\xd6\x95\x52\x90\x85\xd8\x80\xc3\x34\x2a\xa5\x0a\x26\xb6\x14\xb4\x2a\xed\xa4\x07\x47\x1d\xb8\x97\x88\xc9\x5b\x1e\x1d\xfe\xe6\x6a\xb5\xfa\xaf\x22\xf5\xbc\xba\x49\x43\xe1\x64\x07\x0f\x71\x39\x9c\x03\x9a\x64\xa9\x31\x20\xe2\xb0\xb1\xb4\xbe\x1e\x31\x65\xa3\x21\xef\x31\xf4\x20\x63\xc2\x62\xfb\xf7\x9d\x3f\x81\x35\x0b\x48\xbe\x70\xd6\xc3\x80\x5a\x25\x08\xe6\x15\xd3\x3a\x59\xaa\x17\xad\x4b\xdb\xdc\x37\x0f\x37\xf0\x90\x9d\x3b\x90\xb3\xe6\xac\x15\xb8\x37\x38\xf3\xe4\x61\x28\xd7\xe2\xcf\xbb\xa7\xfd\xf1\xb0\x7b\xdc\x6f\xff\x7d\xcc\xef\xcf\xc9\x8c\x94\x64\xfe\xa0\x4d\xab\x37\xed\x1d\xbe\x47\x62\xdc\x9e\xc0\x31\x4e\x06\xa4\x61\x46\x7e\xde\x72\x19\xd6\x75\x8f\x5d\x1e\xaa\x4a\xd0\x47\x07\x82\x23\x8d\xa1\x1c\x73\xc7\xa2\x95\xa4\x8c\xd5\x57\x00\x00\x00\xff\xff\xb2\x3a\x97\xa8\xc3\x01\x00\x00")

func stacksRioAutoscalerStackYamlBytes() ([]byte, error) {
return bindataRead(
Expand Down
2 changes: 1 addition & 1 deletion stacks/rio-autoscaler-stack.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ services:
- '* autoscale.rio.cattle.io/servicescalerecommendations'
- '* rio.cattle.io/services'
- '* rio.cattle.io/services/status'
image: rancher/rio-autoscaler:v0.1.5
image: rancher/rio-autoscaler:v0.1.6
imagePullPolicy: always
env:
- NAMESPACE=${NAMESPACE}
Expand Down