Skip to content
This repository has been archived by the owner on Feb 6, 2025. It is now read-only.

Commit

Permalink
Integrate dex in skuba (#446)
Browse files Browse the repository at this point in the history
* Integrate dex into skuba

Client secret generate by dex, and gangway read the generate client
secret from dex secret resource.

Signed-off-by: JenTing Hsiao <[email protected]>

* Dex is a manatory feature in CaaSP v4, enable OIDC apiserver flag in kubeadm-init.conf.

But still, use could remove the ODIC apiserver flag in kubeadm-init.conf before
skuba node bootstrap.

Signed-off-by: JenTing Hsiao <[email protected]>

* Sign dex/gangway certificate SANs as control plane <IP/FQDN>

Because user access dex/gangway all from control-plane, so sign
SANs same as `skuba cluster init --control-plane <IP/FQDN>` specific.

Signed-off-by: JenTing Hsiao <[email protected]>

* Enable gangway/dex port on load balancer and security group

Developing and testing at OpenStack first, will add VMWare and
other platforms later.

Signed-off-by: JenTing Hsiao <[email protected]>

* Add static user account in dex for testing easily

Signed-off-by: JenTing Hsiao <[email protected]>

* Fix dex client secret cannot get from environment variable

Due to this issue dexidp/dex#1099
client secret is not configurable through environment variable
so, replace client secret in configmap by rendering when skuba
cluster init

Signed-off-by: JenTing Hsiao <[email protected]>

* Renaming openstack load balancer for consistency

Signed-off-by: JenTing Hsiao <[email protected]>

* Skip invalid IP address case

Signed-off-by: JenTing Hsiao <[email protected]>

* Add a comment in dex connector

Signed-off-by: JenTing Hsiao <[email protected]>
  • Loading branch information
jhsiaosuse authored and innobead committed Jul 8, 2019
1 parent 9f1c37a commit 57bd7f4
Show file tree
Hide file tree
Showing 14 changed files with 506 additions and 49 deletions.
80 changes: 72 additions & 8 deletions ci/infra/openstack/load-balancer.tf
Original file line number Diff line number Diff line change
Expand Up @@ -7,35 +7,99 @@ resource "openstack_lb_loadbalancer_v2" "lb" {
]
}

resource "openstack_lb_listener_v2" "listener" {
resource "openstack_lb_listener_v2" "kube_api_listener" {
protocol = "TCP"
protocol_port = "6443"
loadbalancer_id = "${openstack_lb_loadbalancer_v2.lb.id}"
name = "${var.stack_name}-api-server-listener"
name = "${var.stack_name}-kube-api-listener"
}

resource "openstack_lb_pool_v2" "pool" {
resource "openstack_lb_listener_v2" "gangway_listener" {
protocol = "TCP"
protocol_port = "32001"
loadbalancer_id = "${openstack_lb_loadbalancer_v2.lb.id}"
name = "${var.stack_name}-gangway-listener"
}

resource "openstack_lb_listener_v2" "dex_listener" {
protocol = "TCP"
protocol_port = "32002"
loadbalancer_id = "${openstack_lb_loadbalancer_v2.lb.id}"
name = "${var.stack_name}-dex-listener"
}

resource "openstack_lb_pool_v2" "kube_api_pool" {
name = "${var.stack_name}-kube-api-pool"
protocol = "TCP"
lb_method = "ROUND_ROBIN"
listener_id = "${openstack_lb_listener_v2.listener.id}"
listener_id = "${openstack_lb_listener_v2.kube_api_listener.id}"
}

resource "openstack_lb_member_v2" "member" {
resource "openstack_lb_pool_v2" "gangway_pool" {
name = "${var.stack_name}-gangway-pool"
protocol = "TCP"
lb_method = "ROUND_ROBIN"
listener_id = "${openstack_lb_listener_v2.gangway_listener.id}"
}

resource "openstack_lb_pool_v2" "dex_pool" {
name = "${var.stack_name}-dex-pool"
protocol = "TCP"
lb_method = "ROUND_ROBIN"
listener_id = "${openstack_lb_listener_v2.dex_listener.id}"
}

resource "openstack_lb_member_v2" "kube_api_member" {
count = "${var.masters}"
pool_id = "${openstack_lb_pool_v2.pool.id}"
pool_id = "${openstack_lb_pool_v2.kube_api_pool.id}"
address = "${element(openstack_compute_instance_v2.master.*.access_ip_v4, count.index)}"
subnet_id = "${openstack_networking_subnet_v2.subnet.id}"
protocol_port = 6443
}

resource "openstack_lb_member_v2" "gangway_member" {
count = "${var.masters}"
pool_id = "${openstack_lb_pool_v2.gangway_pool.id}"
address = "${element(openstack_compute_instance_v2.master.*.access_ip_v4, count.index)}"
subnet_id = "${openstack_networking_subnet_v2.subnet.id}"
protocol_port = 32001
}

resource "openstack_lb_member_v2" "dex_member" {
count = "${var.masters}"
pool_id = "${openstack_lb_pool_v2.dex_pool.id}"
address = "${element(openstack_compute_instance_v2.master.*.access_ip_v4, count.index)}"
subnet_id = "${openstack_networking_subnet_v2.subnet.id}"
protocol_port = 32002
}

resource "openstack_networking_floatingip_v2" "lb_ext" {
pool = "${var.external_net}"
port_id = "${openstack_lb_loadbalancer_v2.lb.vip_port_id}"
}

resource "openstack_lb_monitor_v2" "monitor" {
pool_id = "${openstack_lb_pool_v2.pool.id}"
resource "openstack_lb_monitor_v2" "kube_api_monitor" {
pool_id = "${openstack_lb_pool_v2.kube_api_pool.id}"
type = "HTTPS"
url_path = "/healthz"
expected_codes = 200
delay = 10
timeout = 5
max_retries = 3
}

resource "openstack_lb_monitor_v2" "gangway_monitor" {
pool_id = "${openstack_lb_pool_v2.gangway_pool.id}"
type = "HTTPS"
url_path = "/"
expected_codes = 200
delay = 10
timeout = 5
max_retries = 3
}

resource "openstack_lb_monitor_v2" "dex_monitor" {
pool_id = "${openstack_lb_pool_v2.dex_pool.id}"
type = "HTTPS"
url_path = "/healthz"
expected_codes = 200
Expand Down
14 changes: 14 additions & 0 deletions ci/infra/openstack/security-groups.tf
Original file line number Diff line number Diff line change
Expand Up @@ -156,4 +156,18 @@ resource "openstack_compute_secgroup_v2" "secgroup_master_lb" {
ip_protocol = "tcp"
cidr = "0.0.0.0/0"
}

rule {
from_port = 32001
to_port = 32001
ip_protocol = "tcp"
cidr = "0.0.0.0/0"
}

rule {
from_port = 32002
to_port = 32002
ip_protocol = "tcp"
cidr = "0.0.0.0/0"
}
}
3 changes: 3 additions & 0 deletions internal/app/skuba/cluster/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"k8s.io/klog"

cilium "github.com/SUSE/skuba/internal/pkg/skuba/cni"
"github.com/SUSE/skuba/internal/pkg/skuba/dex"
"github.com/SUSE/skuba/internal/pkg/skuba/gangway"
"github.com/SUSE/skuba/internal/pkg/skuba/kured"
cluster "github.com/SUSE/skuba/pkg/skuba/actions/cluster/init"
Expand All @@ -45,6 +46,8 @@ func NewInitCmd() *cobra.Command {
CiliumInitImage: cilium.GetCiliumInitImage(),
CiliumOperatorImage: cilium.GetCiliumOperatorImage(),
KuredImage: kured.GetKuredImage(),
DexImage: dex.GetDexImage(),
GangwayClientSecret: dex.GetClientSecretGangway(),
GangwayImage: gangway.GetGangwayImage(),
}

Expand Down
2 changes: 1 addition & 1 deletion internal/pkg/skuba/cni/cilium.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import (
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/images"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
pkiutil "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
"k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"

"github.com/SUSE/skuba/internal/pkg/skuba/kubeadm"
"github.com/SUSE/skuba/internal/pkg/skuba/kubernetes"
Expand Down
54 changes: 54 additions & 0 deletions internal/pkg/skuba/deployments/ssh/dex.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* Copyright (c) 2019 SUSE LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package ssh

import (
"io/ioutil"
"path/filepath"

"github.com/pkg/errors"

"github.com/SUSE/skuba/internal/pkg/skuba/dex"
"github.com/SUSE/skuba/pkg/skuba"
)

func init() {
stateMap["dex.deploy"] = dexDeploy
}

func dexDeploy(t *Target, data interface{}) error {
if err := dex.CreateDexCert(); err != nil {
return errors.Wrap(err, "unable to create dex certificate")
}

dexFiles, err := ioutil.ReadDir(skuba.DexDir())
if err != nil {
return errors.Wrap(err, "could not read local dex directory")
}

defer t.ssh("rm -rf /tmp/dex.d")

for _, f := range dexFiles {
if err := t.target.UploadFile(filepath.Join(skuba.DexDir(), f.Name()), filepath.Join("/tmp/dex.d", f.Name())); err != nil {
return err
}
}

_, _, err = t.ssh("kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /tmp/dex.d")
return err
}
4 changes: 2 additions & 2 deletions internal/pkg/skuba/deployments/ssh/gangway.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ func init() {
}

func gangwayDeploy(t *Target, data interface{}) error {
if err := gangway.CreateGangwaySecret(); err != nil {
return errors.Wrap(err, "unable to create gangway secret")
if err := gangway.CreateGangwaySessionKey(); err != nil {
return errors.Wrap(err, "unable to create gangway session key")
}
if err := gangway.CreateGangwayCert(); err != nil {
return errors.Wrap(err, "unable to create gangway certificate")
Expand Down
125 changes: 125 additions & 0 deletions internal/pkg/skuba/dex/dex.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
/*
* Copyright (c) 2019 SUSE LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package dex

import (
"crypto/rand"
"crypto/x509"
"fmt"
"net"

"github.com/pkg/errors"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
certutil "k8s.io/client-go/util/cert"
"k8s.io/client-go/util/keyutil"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/images"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
"k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"

"github.com/SUSE/skuba/internal/pkg/skuba/kubernetes"
"github.com/SUSE/skuba/pkg/skuba"
node "github.com/SUSE/skuba/pkg/skuba/actions/node/bootstrap"
)

const (
certName = "oidc-dex-cert"
)

// CreateDexCert creates a signed certificate for dex
// with kubernetes CA certificate and key
func CreateDexCert() error {
// Load kubernetes CA
caCert, caKey, err := pkiutil.TryLoadCertAndKeyFromDisk("pki", "ca")
if err != nil {
return errors.Errorf("unable to load kubernetes CA certificate and key %v", err)
}

// Load kubeadm-init.conf to get certificate SANs
cfg, err := node.LoadInitConfigurationFromFile(skuba.KubeadmInitConfFile())
if err != nil {
return errors.Wrapf(err, "could not parse %s file", skuba.KubeadmInitConfFile())
}
certIPs := make([]net.IP, 0)
for _, san := range cfg.ClusterConfiguration.APIServer.CertSANs {
if ip := net.ParseIP(san); ip != nil {
certIPs = append(certIPs, ip)
}
}

// Generate dex certificate
cert, key, err := pkiutil.NewCertAndKey(caCert, caKey, &certutil.Config{
CommonName: "oidc-dex",
Organization: []string{kubeadmconstants.SystemPrivilegedGroup},
AltNames: certutil.AltNames{
DNSNames: cfg.ClusterConfiguration.APIServer.CertSANs,
IPs: certIPs,
},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
})
if err != nil {
return errors.Errorf("error when creating dex certificate %v", err)
}
privateKey, err := keyutil.MarshalPrivateKeyToPEM(key)
if err != nil {
return errors.Errorf("dex private key marshal failed %v", err)
}

// Write certificate into secret resource
secret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: certName,
Namespace: metav1.NamespaceSystem,
},
Type: v1.SecretTypeTLS,
Data: map[string][]byte{
v1.TLSCertKey: pkiutil.EncodeCertPEM(cert),
v1.TLSPrivateKeyKey: privateKey,
v1.ServiceAccountRootCAKey: pkiutil.EncodeCertPEM(caCert),
},
}

client, err := kubernetes.GetAdminClientSet()
if err != nil {
return errors.Wrap(err, "unable to get admin client set")
}
if err = apiclient.CreateOrUpdateSecret(client, secret); err != nil {
return errors.Errorf("error when creating dex secret %v", err)
}

return nil
}

// GetDexImage returns dex image registry
func GetDexImage() string {
return images.GetGenericImage(skuba.ImageRepository, "caasp-dex",
kubernetes.CurrentAddonVersion(kubernetes.Dex))
}

// GetClientSecretGangway returns client secret which is used by
// auth client (gangway) to authenticate to auth server (dex)
//
// Due to this issue https://github.com/dexidp/dex/issues/1099
// client secret is not configurable through environment variable
// so, replace client secret in configmap by rendering
func GetClientSecretGangway() string {
b := make([]byte, 12)
rand.Read(b)
return fmt.Sprintf("%x", b)
}
Loading

0 comments on commit 57bd7f4

Please sign in to comment.