Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add token requests client #805

Merged
merged 5 commits into from
Feb 3, 2022
Merged

feat: add token requests client #805

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d82cd2a
feat: add token requests client
aramase Nov 2, 2021
3b9515c
test: add e2e test with e2e-provider and add helm config
aramase Jan 26, 2022
88eabde
chore: move token request rbac markers to separate pkg
aramase Jan 27, 2022
67d9370
test: use multiple test functions
aramase Jan 27, 2022
604c0b3
chore: set resync duration at caller
aramase Feb 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -404,6 +404,7 @@ e2e-deploy-manifest:
kubectl apply -f manifest_staging/deploy/rbac-secretproviderclass.yaml
kubectl apply -f manifest_staging/deploy/rbac-secretproviderrotation.yaml
kubectl apply -f manifest_staging/deploy/rbac-secretprovidersyncing.yaml
kubectl apply -f manifest_staging/deploy/rbac-secretprovidertokenrequest.yaml
kubectl apply -f manifest_staging/deploy/secrets-store.csi.x-k8s.io_secretproviderclasses.yaml
kubectl apply -f manifest_staging/deploy/secrets-store.csi.x-k8s.io_secretproviderclasspodstatuses.yaml
kubectl apply -f manifest_staging/deploy/role-secretproviderclasses-admin.yaml
Expand All @@ -429,7 +430,9 @@ e2e-helm-deploy:
--set linux.enabled=true \
--set syncSecret.enabled=true \
--set enableSecretRotation=true \
--set rotationPollInterval=30s
--set rotationPollInterval=30s \
--set tokenRequests[0].audience="aud1" \
--set tokenRequests[1].audience="aud2"

.PHONY: e2e-helm-upgrade
e2e-helm-upgrade:
Expand Down Expand Up @@ -517,6 +520,14 @@ manifests: $(CONTROLLER_GEN) $(KUSTOMIZE)
@sed -i '1s/^/{{ if .Values.enableSecretRotation }}\n/gm; $$s/$$/\n{{ end }}/gm' manifest_staging/charts/secrets-store-csi-driver/templates/role-rotation.yaml
@sed -i '1s/^/{{ if .Values.enableSecretRotation }}\n/gm; s/namespace: .*/namespace: {{ .Release.Namespace }}/gm; $$s/$$/\n{{ end }}/gm' manifest_staging/charts/secrets-store-csi-driver/templates/role-rotation_binding.yaml

# Generate token requests specific RBAC
$(CONTROLLER_GEN) rbac:roleName=secretprovidertokenrequest-role paths="./controllers/tokenrequest" output:dir=config/rbac-tokenrequest
$(KUSTOMIZE) build config/rbac-tokenrequest -o manifest_staging/deploy/rbac-secretprovidertokenrequest.yaml
cp config/rbac-tokenrequest/role.yaml manifest_staging/charts/secrets-store-csi-driver/templates/role-tokenrequest.yaml
cp config/rbac-tokenrequest/role_binding.yaml manifest_staging/charts/secrets-store-csi-driver/templates/role-tokenrequest_binding.yaml
@sed -i '1s/^/{{ if .Values.tokenRequests }}\n/gm; $$s/$$/\n{{ end }}/gm' manifest_staging/charts/secrets-store-csi-driver/templates/role-tokenrequest.yaml
@sed -i '1s/^/{{ if .Values.tokenRequests }}\n/gm; s/namespace: .*/namespace: {{ .Release.Namespace }}/gm; $$s/$$/\n{{ end }}/gm' manifest_staging/charts/secrets-store-csi-driver/templates/role-tokenrequest_binding.yaml

.PHONY: generate-protobuf
generate-protobuf: $(PROTOC) $(PROTOC_GEN_GO) # generates protobuf
$(PROTOC) -I . provider/v1alpha1/service.proto --go_out=plugins=grpc:. --plugin=$(PROTOC_GEN_GO)
Expand Down
21 changes: 18 additions & 3 deletions cmd/secrets-store-csi-driver/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (

secretsstorev1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1"
"sigs.k8s.io/secrets-store-csi-driver/controllers"
"sigs.k8s.io/secrets-store-csi-driver/pkg/k8s"
"sigs.k8s.io/secrets-store-csi-driver/pkg/metrics"
"sigs.k8s.io/secrets-store-csi-driver/pkg/rotation"
secretsstore "sigs.k8s.io/secrets-store-csi-driver/pkg/secrets-store"
Expand All @@ -38,6 +39,7 @@ import (
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
json "k8s.io/component-base/logs/json"
Expand Down Expand Up @@ -82,7 +84,8 @@ func main() {
flag.Parse()

if *logFormatJSON {
klog.SetLogger(json.JSONLogger)
logger, _ := json.NewJSONLogger(nil, nil)
klog.SetLogger(logger)
}
if *enableProfile {
klog.InfoS("Starting profiling", "port", *profilePort)
Expand Down Expand Up @@ -177,17 +180,29 @@ func main() {
reconciler.RunPatcher(ctx)
}()

// token request client
kubeClient := kubernetes.NewForConfigOrDie(cfg)
tokenClient := k8s.NewTokenClient(kubeClient, *driverName, 10*time.Minute)
if err != nil {
klog.ErrorS(err, "failed to create token client")
os.Exit(1)
}
if err = tokenClient.Run(ctx.Done()); err != nil {
klog.ErrorS(err, "failed to run token client")
os.Exit(1)
}

// Secret rotation
if *enableSecretRotation {
rec, err := rotation.NewReconciler(mgr.GetCache(), scheme, *providerVolumePath, *nodeID, *rotationPollInterval, providerClients)
rec, err := rotation.NewReconciler(mgr.GetCache(), scheme, *providerVolumePath, *nodeID, *rotationPollInterval, providerClients, tokenClient)
if err != nil {
klog.ErrorS(err, "failed to initialize rotation reconciler")
os.Exit(1)
}
go rec.Run(ctx.Done())
}

driver := secretsstore.NewSecretsStoreDriver(*driverName, *nodeID, *endpoint, *providerVolumePath, providerClients, mgr.GetClient(), mgr.GetAPIReader())
driver := secretsstore.NewSecretsStoreDriver(*driverName, *nodeID, *endpoint, *providerVolumePath, providerClients, mgr.GetClient(), mgr.GetAPIReader(), tokenClient)
driver.Run(ctx)
}

Expand Down
3 changes: 3 additions & 0 deletions config/rbac-tokenrequest/kustomization.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
resources:
- role.yaml
- role_binding.yaml
14 changes: 14 additions & 0 deletions config/rbac-tokenrequest/role.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
creationTimestamp: null
name: secretprovidertokenrequest-role
rules:
- apiGroups:
- ""
resources:
- serviceaccounts/token
verbs:
- create
12 changes: 12 additions & 0 deletions config/rbac-tokenrequest/role_binding.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: secretprovidertokenrequest-rolebinding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: secretprovidertokenrequest-role
subjects:
- kind: ServiceAccount
name: secrets-store-csi-driver
namespace: kube-system
10 changes: 10 additions & 0 deletions config/rbac/role.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,3 +49,13 @@ rules:
- get
- patch
- update
- apiGroups:
- storage.k8s.io
resourceNames:
- secrets-store.csi.k8s.io
resources:
- csidrivers
verbs:
- get
- list
- watch
1 change: 1 addition & 0 deletions controllers/secretproviderclasspodstatus_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,7 @@ func (r *SecretProviderClassPodStatusReconciler) ListOptionsLabelSelector() clie
// +kubebuilder:rbac:groups=secrets-store.csi.x-k8s.io,resources=secretproviderclasses,verbs=get;list;watch
// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch
// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch
// +kubebuilder:rbac:groups="storage.k8s.io",resources=csidrivers,verbs=get;list;watch,resourceNames=secrets-store.csi.k8s.io

func (r *SecretProviderClassPodStatusReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
r.mutex.Lock()
Expand Down
22 changes: 22 additions & 0 deletions controllers/tokenrequest/tokenrequest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/*
Copyright 2022 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package tokenrequest holds the RBAC permission annotations for the controller
// to create a serviceaccount token and pass it as part of Mount Request.
// ref: https://kubernetes-csi.github.io/docs/token-requests.html
package tokenrequest

// +kubebuilder:rbac:groups="",resources="serviceaccounts/token",verbs=create
94 changes: 65 additions & 29 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,46 +8,50 @@ require (
github.com/google/go-cmp v0.5.5
github.com/kubernetes-csi/csi-lib-utils v0.10.0
github.com/kubernetes-csi/csi-test/v4 v4.2.0
github.com/onsi/gomega v1.13.0
github.com/onsi/gomega v1.17.0
github.com/prometheus/client_golang v1.11.0
github.com/stretchr/testify v1.7.0
go.opentelemetry.io/otel v0.20.0
go.opentelemetry.io/otel/exporters/metric/prometheus v0.20.0
go.opentelemetry.io/otel/metric v0.20.0
google.golang.org/grpc v1.39.0
google.golang.org/protobuf v1.26.0
k8s.io/api v0.22.3
k8s.io/apimachinery v0.22.3
k8s.io/client-go v0.22.3
k8s.io/component-base v0.22.3
k8s.io/klog/v2 v2.10.0
k8s.io/mount-utils v0.22.3
sigs.k8s.io/controller-runtime v0.9.0
google.golang.org/grpc v1.40.0
google.golang.org/protobuf v1.27.1
k8s.io/api v0.23.0
k8s.io/apimachinery v0.23.0
k8s.io/client-go v0.23.0
k8s.io/component-base v0.23.0
k8s.io/klog/v2 v2.30.0
k8s.io/kubernetes v1.23.0
k8s.io/mount-utils v0.23.0
sigs.k8s.io/controller-runtime v0.11.0
)

require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/evanphx/json-patch v4.11.0+incompatible // indirect
github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/go-logr/logr v0.4.0 // indirect
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/go-logr/logr v1.2.0 // indirect
github.com/go-logr/zapr v1.2.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/gofuzz v1.1.0 // indirect
github.com/google/uuid v1.1.2 // indirect
github.com/googleapis/gnostic v0.5.5 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/json-iterator/go v1.1.11 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/onsi/ginkgo v1.16.4 // indirect
github.com/onsi/ginkgo v1.16.5 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.26.0 // indirect
github.com/prometheus/common v0.28.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
go.opentelemetry.io/otel/sdk v0.20.0 // indirect
Expand All @@ -56,24 +60,56 @@ require (
go.opentelemetry.io/otel/trace v0.20.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.17.0 // indirect
golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 // indirect
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
golang.org/x/text v0.3.6 // indirect
go.uber.org/zap v1.19.1 // indirect
golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect
golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8 // indirect
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect
google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
k8s.io/apiextensions-apiserver v0.21.1 // indirect
k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e // indirect
k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
sigs.k8s.io/yaml v1.2.0 // indirect
k8s.io/apiextensions-apiserver v0.23.0 // indirect
k8s.io/apiserver v0.23.0 // indirect
k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)

replace (
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i feel like there must be a way to do this without all these replace directives (I'm goin got checkout the branch and try playing around with some things to make go mod happy...)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay so I think i understand this a bit better...

This PR introduces a dependency on https://pkg.go.dev/k8s.io/kubernetes/pkg/kubelet/token

which pulls in the k8s.io/kubernetes v1.23.0 module, but according to kubernetes/kubernetes#90358 (comment)

"depending on k8s.io/kubernetes directly as a library, which is not supported"

I guess what we're doing here is whats mentioned kubernetes/kubernetes#79384

Since this token dependency is not permanent (we have plan to remove and replace with RequiresRepublish once thats supposed by all K8s supported releases) I think I'm good with these replace directives - but we should definitely add a comment explaining that k8s.io/kubernetes dependency causes the problems linking to kubernetes/kubernetes#79384

k8s.io/api => k8s.io/api v0.23.0
k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.23.0
k8s.io/apimachinery => k8s.io/apimachinery v0.23.0
k8s.io/apiserver => k8s.io/apiserver v0.23.0
k8s.io/cli-runtime => k8s.io/cli-runtime v0.23.0
k8s.io/client-go => k8s.io/client-go v0.23.0
k8s.io/cloud-provider => k8s.io/cloud-provider v0.23.0
k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.23.0
k8s.io/code-generator => k8s.io/code-generator v0.23.0
k8s.io/component-base => k8s.io/component-base v0.23.0
k8s.io/component-helpers => k8s.io/component-helpers v0.23.0
k8s.io/controller-manager => k8s.io/controller-manager v0.23.0
k8s.io/cri-api => k8s.io/cri-api v0.23.0
k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.23.0
k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.23.0
k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.23.0
k8s.io/kube-proxy => k8s.io/kube-proxy v0.23.0
k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.23.0
k8s.io/kubectl => k8s.io/kubectl v0.23.0
k8s.io/kubelet => k8s.io/kubelet v0.23.0
k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.23.0
k8s.io/metrics => k8s.io/metrics v0.23.0
k8s.io/mount-utils => k8s.io/mount-utils v0.23.0
k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.23.0
k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.23.0
k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.23.0
k8s.io/sample-controller => k8s.io/sample-controller v0.23.0
)
Loading