Merge pull request #5015 from laozc/populate-ca

Support adding untrusted root CA certificates (corp certs)

Author: tstromberg

deploy/iso/minikube-iso/configs/minikube_defconfig

@@ -47,6 +47,8 @@ BR2_PACKAGE_SOCAT=y
 BR2_PACKAGE_SUDO=y
 BR2_PACKAGE_ACL=y
 BR2_PACKAGE_COREUTILS=y
+BR2_PACKAGE_LIBRESSL=y
+BR2_PACKAGE_LIBRESSL_BIN=y
 BR2_PACKAGE_OPENVMTOOLS=y
 BR2_PACKAGE_OPENVMTOOLS_PROCPS=y
 BR2_PACKAGE_SYSTEMD_LOGIND=y

pkg/minikube/bootstrapper/certs.go

@@ -17,8 +17,11 @@ limitations under the License.
 package bootstrapper
 
 import (
+	"encoding/pem"
 	"fmt"
+	"io/ioutil"
 	"net"
+	"os"
 	"path"
 	"path/filepath"
 	"strings"
@@ -36,6 +39,11 @@ import (
 	"k8s.io/minikube/pkg/util"
 )
 
+const (
+	CACertificatesDir = "/usr/share/ca-certificates"
+	SSLCertStoreDir   = "/etc/ssl/certs"
+)
+
 var (
 	certs = []string{
 		"ca.crt", "ca.key", "apiserver.crt", "apiserver.key", "proxy-client-ca.crt",
@@ -67,6 +75,19 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error {
 		copyableFiles = append(copyableFiles, certFile)
 	}
 
+	caCerts, err := collectCACerts()
+	if err != nil {
+		return err
+	}
+	for src, dst := range caCerts {
+		certFile, err := assets.NewFileAsset(src, path.Dir(dst), path.Base(dst), "0644")
+		if err != nil {
+			return err
+		}
+
+		copyableFiles = append(copyableFiles, certFile)
+	}
+
 	kcs := &kubeconfig.Settings{
 		ClusterName:          k8s.NodeName,
 		ClusterServerAddress: fmt.Sprintf("https://localhost:%d", k8s.NodePort),
@@ -77,7 +98,7 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error {
 	}
 
 	kubeCfg := api.NewConfig()
-	err := kubeconfig.PopulateFromSettings(kcs, kubeCfg)
+	err = kubeconfig.PopulateFromSettings(kcs, kubeCfg)
 	if err != nil {
 		return errors.Wrap(err, "populating kubeconfig")
 	}
@@ -95,6 +116,11 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error {
 			return err
 		}
 	}
+
+	// configure CA certificates
+	if err := configureCACerts(cmd, caCerts); err != nil {
+		return errors.Wrapf(err, "error configuring CA certificates during provisioning %v", err)
+	}
 	return nil
 }
 
@@ -198,3 +224,122 @@ func generateCerts(k8s config.KubernetesConfig) error {
 
 	return nil
 }
+
+// isValidPEMCertificate checks whether the input file is a valid PEM certificate (with at least one CERTIFICATE block)
+func isValidPEMCertificate(filePath string) (bool, error) {
+	fileBytes, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return false, err
+	}
+
+	for {
+		block, rest := pem.Decode(fileBytes)
+		if block == nil {
+			break
+		}
+
+		if block.Type == "CERTIFICATE" {
+			// certificate found
+			return true, nil
+		}
+		fileBytes = rest
+	}
+
+	return false, nil
+}
+
+// collectCACerts looks up all PEM certificates with .crt or .pem extension in ~/.minikube/certs to copy to the host.
+// minikube root CA is also included but libmachine certificates (ca.pem/cert.pem) are excluded.
+func collectCACerts() (map[string]string, error) {
+	localPath := constants.GetMinipath()
+	certFiles := map[string]string{}
+
+	certsDir := filepath.Join(localPath, "certs")
+	err := filepath.Walk(certsDir, func(hostpath string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if info != nil && !info.IsDir() {
+			ext := strings.ToLower(filepath.Ext(hostpath))
+			if ext == ".crt" || ext == ".pem" {
+				validPem, err := isValidPEMCertificate(hostpath)
+				if err != nil {
+					return err
+				}
+				if validPem {
+					filename := filepath.Base(hostpath)
+					dst := fmt.Sprintf("%s.%s", strings.TrimSuffix(filename, ext), "pem")
+					certFiles[hostpath] = path.Join(CACertificatesDir, dst)
+				}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, errors.Wrapf(err, "provisioning: traversal certificates dir %s", certsDir)
+	}
+
+	for _, excluded := range []string{"ca.pem", "cert.pem"} {
+		certFiles[filepath.Join(certsDir, excluded)] = ""
+	}
+
+	// populates minikube CA
+	certFiles[filepath.Join(localPath, "ca.crt")] = path.Join(CACertificatesDir, "minikubeCA.pem")
+
+	filtered := map[string]string{}
+	for k, v := range certFiles {
+		if v != "" {
+			filtered[k] = v
+		}
+	}
+	return filtered, nil
+}
+
+// getSubjectHash calculates Certificate Subject Hash for creating certificate symlinks
+func getSubjectHash(cmd command.Runner, filePath string) (string, error) {
+	out, err := cmd.CombinedOutput(fmt.Sprintf("openssl x509 -hash -noout -in '%s'", filePath))
+	if err != nil {
+		return "", err
+	}
+
+	stringHash := strings.TrimSpace(out)
+	return stringHash, nil
+}
+
+// configureCACerts looks up and installs all uploaded PEM certificates in /usr/share/ca-certificates to system-wide certificate store (/etc/ssl/certs).
+// OpenSSL binary required in minikube ISO
+func configureCACerts(cmd command.Runner, caCerts map[string]string) error {
+	hasSSLBinary := true
+	if err := cmd.Run("which openssl"); err != nil {
+		hasSSLBinary = false
+	}
+
+	if !hasSSLBinary && len(caCerts) > 0 {
+		glog.Warning("OpenSSL not found. Please recreate the cluster with the latest minikube ISO.")
+	}
+
+	for _, caCertFile := range caCerts {
+		dstFilename := path.Base(caCertFile)
+		certStorePath := path.Join(SSLCertStoreDir, dstFilename)
+		if err := cmd.Run(fmt.Sprintf("sudo test -f '%s'", certStorePath)); err != nil {
+			if err := cmd.Run(fmt.Sprintf("sudo ln -s '%s' '%s'", caCertFile, certStorePath)); err != nil {
+				return errors.Wrapf(err, "error making symbol link for certificate %s", caCertFile)
+			}
+		}
+		if hasSSLBinary {
+			subjectHash, err := getSubjectHash(cmd, caCertFile)
+			if err != nil {
+				return errors.Wrapf(err, "error calculating subject hash for certificate %s", caCertFile)
+			}
+			subjectHashLink := path.Join(SSLCertStoreDir, fmt.Sprintf("%s.0", subjectHash))
+			if err := cmd.Run(fmt.Sprintf("sudo test -f '%s'", subjectHashLink)); err != nil {
+				if err := cmd.Run(fmt.Sprintf("sudo ln -s '%s' '%s'", certStorePath, subjectHashLink)); err != nil {
+					return errors.Wrapf(err, "error making subject hash symbol link for certificate %s", caCertFile)
+				}
+			}
+		}
+	}
+
+	return nil
+}

pkg/minikube/bootstrapper/certs_test.go

@@ -17,7 +17,9 @@ limitations under the License.
 package bootstrapper
 
 import (
+	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 	"testing"
 
@@ -39,10 +41,37 @@ func TestSetupCerts(t *testing.T) {
 		ServiceCIDR:   util.DefaultServiceCIDR,
 	}
 
+	if err := os.Mkdir(filepath.Join(tempDir, "certs"), 0777); err != nil {
+		t.Fatalf("error create certificate directory: %v", err)
+	}
+
+	if err := util.GenerateCACert(
+		filepath.Join(tempDir, "certs", "mycert.pem"),
+		filepath.Join(tempDir, "certs", "mykey.pem"),
+		"Test Certificate",
+	); err != nil {
+		t.Fatalf("error generating certificate: %v", err)
+	}
+
+	cmdMap := map[string]string{}
+	certFilenames := map[string]string{"ca.crt": "minikubeCA.pem", "mycert.pem": "mycert.pem"}
+	for _, dst := range certFilenames {
+		certFile := path.Join(CACertificatesDir, dst)
+		certStorePath := path.Join(SSLCertStoreDir, dst)
+		certNameHash := "abcdef"
+		remoteCertHashLink := path.Join(SSLCertStoreDir, fmt.Sprintf("%s.0", certNameHash))
+		cmdMap[fmt.Sprintf("sudo ln -s '%s' '%s'", certFile, certStorePath)] = "1"
+		cmdMap[fmt.Sprintf("openssl x509 -hash -noout -in '%s'", certFile)] = certNameHash
+		cmdMap[fmt.Sprintf("sudo ln -s '%s' '%s'", certStorePath, remoteCertHashLink)] = "1"
+	}
+	f.SetCommandToOutput(cmdMap)
+
 	var filesToBeTransferred []string
 	for _, cert := range certs {
 		filesToBeTransferred = append(filesToBeTransferred, filepath.Join(constants.GetMinipath(), cert))
 	}
+	filesToBeTransferred = append(filesToBeTransferred, filepath.Join(constants.GetMinipath(), "ca.crt"))
+	filesToBeTransferred = append(filesToBeTransferred, filepath.Join(constants.GetMinipath(), "certs", "mycert.pem"))
 
 	if err := SetupCerts(f, k8s); err != nil {
 		t.Fatalf("Error starting cluster: %v", err)

site/content/en/docs/Tutorials/untrusted_root_certificate.md

@@ -0,0 +1,42 @@
+---
+title: "Untrusted Root Certificate"
+linkTitle: "Untrusted Root Certificate"
+weight: 1
+date: 2019-08-15
+description: >
+  Using minikube with Untrusted Root Certificate
+---
+
+## Overview
+
+Most organizations deploy their own Root Certificate and CA service inside the corporate networks.
+Internal websites, image repositories and other resources may install SSL server certificates issued by this CA service for security and privacy concerns.
+You may install the Root Certificate into the minikube VM to access these corporate resources within the cluster.
+
+## Prerequisites
+
+- Corporate X.509 Root Certificate
+- Latest minikube binary and ISO
+
+## Tutorial
+
+* The certificate must be in PEM format. You may use `openssl` to convert from DER format.
+
+```
+openssl x509 -inform der -in my_company.cer -out my_company.pem
+```
+
+* You may need to delete existing minikube VM
+
+```shell
+minikube delete
+```
+
+* Copy the certificate before creating the minikube VM
+
+```shell
+mkdir -p $HOME/.minikube/certs
+cp my_company.pem $HOME/.minikube/certs/my_company.pem
+
+minikube start
+```