Populates CA certificates

This PR allows users to add root CA certificates in minikube
VM. CA certificates in $HOME/.minikube/certs will be populated
to system certificate store.

Note: This requires a change to minikube ISO so you may need to
delete and start a brand new minikube VM.

Signed-off-by: Zhongcheng Lao <[email protected]>

Author: laozc

pkg/minikube/bootstrapper/certs.go

@@ -17,8 +17,11 @@ limitations under the License.
 package bootstrapper
 
 import (
+	"encoding/pem"
 	"fmt"
+	"io/ioutil"
 	"net"
+	"os"
 	"path"
 	"path/filepath"
 	"strings"
@@ -66,6 +69,19 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error {
 		copyableFiles = append(copyableFiles, certFile)
 	}
 
+	caCerts, err := collectCACerts()
+	if err != nil {
+		return err
+	}
+	for src, dst := range caCerts {
+		certFile, err := assets.NewFileAsset(src, path.Dir(dst), path.Base(dst), "0644")
+		if err != nil {
+			return err
+		}
+
+		copyableFiles = append(copyableFiles, certFile)
+	}
+
 	kubeCfgSetup := &util.KubeConfigSetup{
 		ClusterName:          k8s.NodeName,
 		ClusterServerAddress: fmt.Sprintf("https://localhost:%d", k8s.NodePort),
@@ -76,7 +92,7 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error {
 	}
 
 	kubeCfg := api.NewConfig()
-	err := util.PopulateKubeConfig(kubeCfgSetup, kubeCfg)
+	err = util.PopulateKubeConfig(kubeCfgSetup, kubeCfg)
 	if err != nil {
 		return errors.Wrap(err, "populating kubeconfig")
 	}
@@ -94,6 +110,11 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error {
 			return err
 		}
 	}
+
+	// configure CA certificates
+	if err := configureCACerts(cmd, caCerts); err != nil {
+		return errors.Wrapf(err, "error configuring CA certificates during provisioning %v", err)
+	}
 	return nil
 }
 
@@ -197,3 +218,102 @@ func generateCerts(k8s config.KubernetesConfig) error {
 
 	return nil
 }
+
+func collectCACerts() (map[string]string, error) {
+	localPath := constants.GetMinipath()
+
+	isValidPem := func(hostpath string) (bool, error) {
+		fileBytes, err := ioutil.ReadFile(hostpath)
+		if err != nil {
+			return false, err
+		}
+
+		for {
+			block, rest := pem.Decode(fileBytes)
+			if block == nil {
+				break
+			}
+
+			if block.Type == "CERTIFICATE" {
+				// certificate found
+				return true, nil
+			}
+			fileBytes = rest
+		}
+
+		return false, nil
+	}
+
+	certFiles := map[string]string{}
+
+	certsDir := filepath.Join(localPath, "certs")
+	err := filepath.Walk(certsDir, func(hostpath string, info os.FileInfo, err error) error {
+		if info != nil && !info.IsDir() {
+			ext := strings.ToLower(filepath.Ext(hostpath))
+			if ext == ".crt" || ext == ".pem" {
+				validPem, err := isValidPem(hostpath)
+				if err != nil {
+					return err
+				}
+				if validPem {
+					filename := filepath.Base(hostpath)
+					dst := fmt.Sprintf("%s.%s", strings.TrimSuffix(filename, ext), "pem")
+					certFiles[hostpath] = path.Join(constants.CACertificatesDir, dst)
+				}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, errors.Wrapf(err, "provisioning: traversal certificates dir %s", certsDir)
+	}
+
+	for _, excluded := range []string{"ca.pem", "cert.pem"} {
+		certFiles[filepath.Join(certsDir, excluded)] = ""
+	}
+
+	// populates minikube CA
+	certFiles[filepath.Join(localPath, "ca.crt")] = path.Join(constants.CACertificatesDir, "minikubeCA.pem")
+
+	filtered := map[string]string{}
+	for k, v := range certFiles {
+		if v != "" {
+			filtered[k] = v
+		}
+	}
+	return filtered, nil
+}
+
+func configureCACerts(cmd command.Runner, caCerts map[string]string) error {
+	getSubjectHash := func(hostpath string) (string, error) {
+		out, err := cmd.CombinedOutput(fmt.Sprintf("openssl x509 -hash -noout -in '%s'", hostpath))
+		if err != nil {
+			return "", err
+		}
+
+		stringHash := strings.ReplaceAll(out, "\n", "")
+		return stringHash, nil
+	}
+
+	for _, caCertFile := range caCerts {
+		dstFilename := path.Base(caCertFile)
+		certStorePath := path.Join(constants.SSLCertStoreDir, dstFilename)
+		if err := cmd.Run(fmt.Sprintf("sudo test -f '%s'", certStorePath)); err != nil {
+			if err := cmd.Run(fmt.Sprintf("sudo ln -s '%s' '%s'", caCertFile, certStorePath)); err != nil {
+				return errors.Wrapf(err, "error making symbol link for certificate %s", caCertFile)
+			}
+		}
+		subjectHash, err := getSubjectHash(caCertFile)
+		if err != nil {
+			return errors.Wrapf(err, "error calculating subject hash for certificate %s", caCertFile)
+		}
+		subjectHashLink := path.Join(constants.SSLCertStoreDir, fmt.Sprintf("%s.0", subjectHash))
+		if err := cmd.Run(fmt.Sprintf("sudo test -f '%s'", subjectHashLink)); err != nil {
+			if err := cmd.Run(fmt.Sprintf("sudo ln -s '%s' '%s'", certStorePath, subjectHashLink)); err != nil {
+				return errors.Wrapf(err, "error making subject hash symbol link for certificate %s", caCertFile)
+			}
+		}
+	}
+
+	return nil
+}

pkg/minikube/bootstrapper/certs_test.go

@@ -17,7 +17,9 @@ limitations under the License.
 package bootstrapper
 
 import (
+	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 	"testing"
 
@@ -39,10 +41,37 @@ func TestSetupCerts(t *testing.T) {
 		ServiceCIDR:   util.DefaultServiceCIDR,
 	}
 
+	if err := os.Mkdir(filepath.Join(tempDir, "certs"), 0777); err != nil {
+		t.Fatalf("error create certificate directory: %v", err)
+	}
+
+	if err := util.GenerateCACert(
+		filepath.Join(tempDir, "certs", "mycert.pem"),
+		filepath.Join(tempDir, "certs", "mykey.pem"),
+		"Test Certificate",
+	); err != nil {
+		t.Fatalf("error generating certificate: %v", err)
+	}
+
+	cmdMap := map[string]string{}
+	certFilenames := map[string]string{"ca.crt": "minikubeCA.pem", "mycert.pem": "mycert.pem"}
+	for _, dst := range certFilenames {
+		certFile := path.Join(constants.CACertificatesDir, dst)
+		certStorePath := path.Join(constants.SSLCertStoreDir, dst)
+		certNameHash := "abcdef"
+		remoteCertHashLink := path.Join(constants.SSLCertStoreDir, fmt.Sprintf("%s.0", certNameHash))
+		cmdMap[fmt.Sprintf("sudo ln -s '%s' '%s'", certFile, certStorePath)] = "1"
+		cmdMap[fmt.Sprintf("openssl x509 -hash -noout -in '%s'", certFile)] = certNameHash
+		cmdMap[fmt.Sprintf("sudo ln -s '%s' '%s'", certStorePath, remoteCertHashLink)] = "1"
+	}
+	f.SetCommandToOutput(cmdMap)
+
 	var filesToBeTransferred []string
 	for _, cert := range certs {
 		filesToBeTransferred = append(filesToBeTransferred, filepath.Join(constants.GetMinipath(), cert))
 	}
+	filesToBeTransferred = append(filesToBeTransferred, filepath.Join(constants.GetMinipath(), "ca.crt"))
+	filesToBeTransferred = append(filesToBeTransferred, filepath.Join(constants.GetMinipath(), "certs", "mycert.pem"))
 
 	if err := SetupCerts(f, k8s); err != nil {
 		t.Fatalf("Error starting cluster: %v", err)

pkg/minikube/constants/constants.go

@@ -426,3 +426,8 @@ const (
 	// DriverDocumentation the documentation of the KVM driver
 	DriverDocumentation = "https://github.com/kubernetes/minikube/blob/master/docs/drivers.md"
 )
+
+const (
+	CACertificatesDir = "/usr/share/ca-certificates"
+	SSLCertStoreDir   = "/etc/ssl/certs"
+)