Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Fix various panics #603

Merged
merged 2 commits into from
Aug 3, 2020
Merged

fix: Fix various panics #603

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b532014
Fix panic in rollout restart
dthomson25 Jul 31, 2020
f05223e
Fix panic in metrics package
dthomson25 Jul 31, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions controller/metrics/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ var (
)

// IncKubernetesRequest increments the kubernetes client counter
func (m *K8sRequestsCountProvider) IncKubernetesRequest(resourceInfo kubeclientmetrics.ResourceInfo) error {
func (m *K8sRequestsCountProvider) IncKubernetesRequest(resourceInfo kubeclientmetrics.ResourceInfo) {
name := resourceInfo.Name
namespace := resourceInfo.Namespace
kind := resourceInfo.Kind
Expand All @@ -49,5 +49,4 @@ func (m *K8sRequestsCountProvider) IncKubernetesRequest(resourceInfo kubeclientm
}

m.k8sRequestsCount.WithLabelValues(kind, namespace, name, string(resourceInfo.Verb), statusCode).Inc()
return nil
}
8 changes: 2 additions & 6 deletions controller/metrics/client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ package metrics
import (
"testing"

"github.com/stretchr/testify/assert"

"github.com/argoproj/argo-rollouts/utils/kubeclientmetrics"
)

Expand All @@ -21,18 +19,16 @@ func TestIncKubernetesRequest(t *testing.T) {
AnalysisRunLister: fakeAnalysisRunLister{},
K8SRequestProvider: provider,
})
err := provider.IncKubernetesRequest(kubeclientmetrics.ResourceInfo{
provider.IncKubernetesRequest(kubeclientmetrics.ResourceInfo{
Kind: "replicasets",
Namespace: "default",
Name: "test",
Verb: kubeclientmetrics.List,
StatusCode: 200,
})
assert.Nil(t, err)
err = provider.IncKubernetesRequest(kubeclientmetrics.ResourceInfo{
provider.IncKubernetesRequest(kubeclientmetrics.ResourceInfo{
Verb: kubeclientmetrics.Unknown,
StatusCode: 200,
})
assert.Nil(t, err)
testHttpResponse(t, metricsServ.Handler, expectedKubernetesRequest)
}
2 changes: 1 addition & 1 deletion ingress/ingress.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,8 @@ func (c *Controller) syncIngress(key string) error {
if !strings.HasSuffix(name, ingressutil.CanaryIngressSuffix) {
// a primary ingress was deleted, simply ignore the event
log.WithField(logutil.IngressKey, key).Warn("primary ingress has been deleted")
return nil
}
return nil
}
rollouts, err := c.getRolloutsByIngress(ingress.Namespace, ingress.Name)
if err != nil {
Expand Down
5 changes: 4 additions & 1 deletion rollout/restart.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"sort"
"time"

"github.com/argoproj/argo-rollouts/utils/defaults"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -57,7 +59,8 @@ func (p *RolloutPodRestarter) Reconcile(roCtx rolloutContext) error {
logCtx.Info("Reconcile pod restarts")
s := NewSortReplicaSetsByPriority(roCtx)
for _, rs := range s.allRSs {
if rs.Status.AvailableReplicas != *rs.Spec.Replicas {
rsReplicas := defaults.GetReplicasOrDefault(rs.Spec.Replicas)
if rs.Status.AvailableReplicas != rsReplicas {
logCtx.WithField("ReplicaSet", rs.Name).Info("cannot restart pods as not all ReplicasSets are fully available")
return nil
}
Expand Down
11 changes: 9 additions & 2 deletions utils/kubeclientmetrics/metric.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package kubeclientmetrics

import (
"fmt"
"io/ioutil"
"net/http"
"path"
Expand Down Expand Up @@ -41,7 +42,7 @@ func (ri ResourceInfo) HasAllFields() bool {

type metricsRoundTripper struct {
roundTripper http.RoundTripper
inc func(ResourceInfo) error
inc func(ResourceInfo)
processPath *regexp.Regexp
}

Expand Down Expand Up @@ -184,6 +185,12 @@ func handleUpdate(r *http.Request, statusCode int) ResourceInfo {

func (mrt *metricsRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
resp, roundTimeErr := mrt.roundTripper.RoundTrip(r)
if roundTimeErr != nil {
return resp, roundTimeErr
}
if resp == nil {
return nil, fmt.Errorf("round tripper has no response when there is no error. please file an issue at https://github.com/argoproj/argo-rollouts")
}
var info ResourceInfo
switch verb := mrt.resolveK8sRequestVerb(r); verb {
case List:
Expand All @@ -210,7 +217,7 @@ func (mrt *metricsRoundTripper) RoundTrip(r *http.Request) (*http.Response, erro
}

// AddMetricsTransportWrapper adds a transport wrapper which wraps a function call around each kubernetes request
func AddMetricsTransportWrapper(config *rest.Config, incFunc func(ResourceInfo) error) *rest.Config {
func AddMetricsTransportWrapper(config *rest.Config, incFunc func(ResourceInfo)) *rest.Config {
regex := regexp.MustCompile(findPathRegex)
wrap := config.WrapTransport
config.WrapTransport = func(rt http.RoundTripper) http.RoundTripper {
Expand Down
24 changes: 8 additions & 16 deletions utils/kubeclientmetrics/metric_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,8 @@ func TestAddMetricsTransportWrapperWrapTwice(t *testing.T) {
}
}

newConfig := AddMetricsTransportWrapper(config, func(info ResourceInfo) error {
newConfig := AddMetricsTransportWrapper(config, func(info ResourceInfo) {
currentCount++
return nil
})

client := kubernetes.NewForConfigOrDie(newConfig)
Expand Down Expand Up @@ -133,14 +132,13 @@ func TestGetRequest(t *testing.T) {
config := &rest.Config{
Host: ts.URL,
}
newConfig := AddMetricsTransportWrapper(config, func(info ResourceInfo) error {
newConfig := AddMetricsTransportWrapper(config, func(info ResourceInfo) {
assert.Equal(t, expectedStatusCode, info.StatusCode)
assert.Equal(t, "replicasets", info.Kind)
assert.Equal(t, metav1.NamespaceDefault, info.Namespace)
assert.Equal(t, "test", info.Name)
assert.Equal(t, Get, info.Verb)
executed = true
return nil
})
client := kubernetes.NewForConfigOrDie(newConfig)
client.AppsV1().ReplicaSets(metav1.NamespaceDefault).Get("test", metav1.GetOptions{})
Expand All @@ -157,14 +155,13 @@ func TestListRequest(t *testing.T) {
config := &rest.Config{
Host: ts.URL,
}
newConfig := AddMetricsTransportWrapper(config, func(info ResourceInfo) error {
newConfig := AddMetricsTransportWrapper(config, func(info ResourceInfo) {
assert.Equal(t, expectedStatusCode, info.StatusCode)
assert.Equal(t, "replicasets", info.Kind)
assert.Equal(t, metav1.NamespaceDefault, info.Namespace)
assert.Equal(t, "", info.Name)
assert.Equal(t, List, info.Verb)
executed = true
return nil
})
client := kubernetes.NewForConfigOrDie(newConfig)
client.AppsV1().ReplicaSets(metav1.NamespaceDefault).List(metav1.ListOptions{})
Expand All @@ -181,14 +178,13 @@ func TestCreateRequest(t *testing.T) {
config := &rest.Config{
Host: ts.URL,
}
newConfig := AddMetricsTransportWrapper(config, func(info ResourceInfo) error {
newConfig := AddMetricsTransportWrapper(config, func(info ResourceInfo) {
assert.Equal(t, expectedStatusCode, info.StatusCode)
assert.Equal(t, "replicasets", info.Kind)
assert.Equal(t, metav1.NamespaceDefault, info.Namespace)
assert.Equal(t, "test", info.Name)
assert.Equal(t, Create, info.Verb)
executed = true
return nil
})
client := kubernetes.NewForConfigOrDie(newConfig)
rs := &appsv1.ReplicaSet{
Expand All @@ -211,14 +207,13 @@ func TestDeleteRequest(t *testing.T) {
config := &rest.Config{
Host: ts.URL,
}
newConfig := AddMetricsTransportWrapper(config, func(info ResourceInfo) error {
newConfig := AddMetricsTransportWrapper(config, func(info ResourceInfo) {
assert.Equal(t, expectedStatusCode, info.StatusCode)
assert.Equal(t, "replicasets", info.Kind)
assert.Equal(t, metav1.NamespaceDefault, info.Namespace)
assert.Equal(t, "test", info.Name)
assert.Equal(t, Delete, info.Verb)
executed = true
return nil
})
client := kubernetes.NewForConfigOrDie(newConfig)
client.AppsV1().ReplicaSets(metav1.NamespaceDefault).Delete("test", &metav1.DeleteOptions{})
Expand All @@ -235,14 +230,13 @@ func TestPatchRequest(t *testing.T) {
config := &rest.Config{
Host: ts.URL,
}
newConfig := AddMetricsTransportWrapper(config, func(info ResourceInfo) error {
newConfig := AddMetricsTransportWrapper(config, func(info ResourceInfo) {
assert.Equal(t, expectedStatusCode, info.StatusCode)
assert.Equal(t, "replicasets", info.Kind)
assert.Equal(t, metav1.NamespaceDefault, info.Namespace)
assert.Equal(t, "test", info.Name)
assert.Equal(t, Patch, info.Verb)
executed = true
return nil
})
client := kubernetes.NewForConfigOrDie(newConfig)
client.AppsV1().ReplicaSets(metav1.NamespaceDefault).Patch("test", types.MergePatchType, []byte("{}"))
Expand All @@ -259,14 +253,13 @@ func TestUpdateRequest(t *testing.T) {
config := &rest.Config{
Host: ts.URL,
}
newConfig := AddMetricsTransportWrapper(config, func(info ResourceInfo) error {
newConfig := AddMetricsTransportWrapper(config, func(info ResourceInfo) {
assert.Equal(t, expectedStatusCode, info.StatusCode)
assert.Equal(t, "replicasets", info.Kind)
assert.Equal(t, metav1.NamespaceDefault, info.Namespace)
assert.Equal(t, "test", info.Name)
assert.Equal(t, Update, info.Verb)
executed = true
return nil
})
client := kubernetes.NewForConfigOrDie(newConfig)
rs := &appsv1.ReplicaSet{
Expand All @@ -288,11 +281,10 @@ func TestUnknownRequest(t *testing.T) {
config := &rest.Config{
Host: ts.URL,
}
newConfig := AddMetricsTransportWrapper(config, func(info ResourceInfo) error {
newConfig := AddMetricsTransportWrapper(config, func(info ResourceInfo) {
assert.Equal(t, expectedStatusCode, info.StatusCode)
assert.Equal(t, Unknown, info.Verb)
executed = true
return nil
})
client := kubernetes.NewForConfigOrDie(newConfig)
client.Discovery().RESTClient().Verb("invalid-verb").Do()
Expand Down