From a7500b3a32bdb5a2562c6067adec3064da214418 Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Sun, 4 Aug 2019 20:23:23 +0800 Subject: [PATCH 1/6] Adds OpenSSL lib and binaries to minikube ISO Signed-off-by: Zhongcheng Lao --- deploy/iso/minikube-iso/configs/minikube_defconfig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/iso/minikube-iso/configs/minikube_defconfig b/deploy/iso/minikube-iso/configs/minikube_defconfig index ab24b3652a3c..a926ece5ae57 100644 --- a/deploy/iso/minikube-iso/configs/minikube_defconfig +++ b/deploy/iso/minikube-iso/configs/minikube_defconfig @@ -47,6 +47,8 @@ BR2_PACKAGE_SOCAT=y BR2_PACKAGE_SUDO=y BR2_PACKAGE_ACL=y BR2_PACKAGE_COREUTILS=y +BR2_PACKAGE_LIBRESSL=y +BR2_PACKAGE_LIBRESSL_BIN=y BR2_PACKAGE_OPENVMTOOLS=y BR2_PACKAGE_OPENVMTOOLS_PROCPS=y BR2_PACKAGE_SYSTEMD_LOGIND=y From aec9406c623d652284e25f0e53c975ebb8e4860e Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Mon, 5 Aug 2019 00:33:19 +0800 Subject: [PATCH 2/6] Populates CA certificates This PR allows users to add root CA certificates in minikube VM. CA certificates in $HOME/.minikube/certs will be populated to system certificate store. Note: This requires a change to minikube ISO so you may need to delete and start a brand new minikube VM. Signed-off-by: Zhongcheng Lao --- pkg/minikube/bootstrapper/certs.go | 122 +++++++++++++++++++++++- pkg/minikube/bootstrapper/certs_test.go | 29 ++++++ pkg/minikube/constants/constants.go | 5 + 3 files changed, 155 insertions(+), 1 deletion(-) diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go index b24f2e7d830b..10d26faae719 100644 --- a/pkg/minikube/bootstrapper/certs.go +++ b/pkg/minikube/bootstrapper/certs.go @@ -17,8 +17,11 @@ limitations under the License. package bootstrapper import ( + "encoding/pem" "fmt" + "io/ioutil" "net" + "os" "path" "path/filepath" "strings" @@ -66,6 +69,19 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error { copyableFiles = append(copyableFiles, certFile) } + caCerts, err := collectCACerts() + if err != nil { + return err + } + for src, dst := range caCerts { + certFile, err := assets.NewFileAsset(src, path.Dir(dst), path.Base(dst), "0644") + if err != nil { + return err + } + + copyableFiles = append(copyableFiles, certFile) + } + kubeCfgSetup := &util.KubeConfigSetup{ ClusterName: k8s.NodeName, ClusterServerAddress: fmt.Sprintf("https://localhost:%d", k8s.NodePort), @@ -76,7 +92,7 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error { } kubeCfg := api.NewConfig() - err := util.PopulateKubeConfig(kubeCfgSetup, kubeCfg) + err = util.PopulateKubeConfig(kubeCfgSetup, kubeCfg) if err != nil { return errors.Wrap(err, "populating kubeconfig") } @@ -94,6 +110,11 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error { return err } } + + // configure CA certificates + if err := configureCACerts(cmd, caCerts); err != nil { + return errors.Wrapf(err, "error configuring CA certificates during provisioning %v", err) + } return nil } @@ -197,3 +218,102 @@ func generateCerts(k8s config.KubernetesConfig) error { return nil } + +func collectCACerts() (map[string]string, error) { + localPath := constants.GetMinipath() + + isValidPem := func(hostpath string) (bool, error) { + fileBytes, err := ioutil.ReadFile(hostpath) + if err != nil { + return false, err + } + + for { + block, rest := pem.Decode(fileBytes) + if block == nil { + break + } + + if block.Type == "CERTIFICATE" { + // certificate found + return true, nil + } + fileBytes = rest + } + + return false, nil + } + + certFiles := map[string]string{} + + certsDir := filepath.Join(localPath, "certs") + err := filepath.Walk(certsDir, func(hostpath string, info os.FileInfo, err error) error { + if info != nil && !info.IsDir() { + ext := strings.ToLower(filepath.Ext(hostpath)) + if ext == ".crt" || ext == ".pem" { + validPem, err := isValidPem(hostpath) + if err != nil { + return err + } + if validPem { + filename := filepath.Base(hostpath) + dst := fmt.Sprintf("%s.%s", strings.TrimSuffix(filename, ext), "pem") + certFiles[hostpath] = path.Join(constants.CACertificatesDir, dst) + } + } + } + return nil + }) + if err != nil { + return nil, errors.Wrapf(err, "provisioning: traversal certificates dir %s", certsDir) + } + + for _, excluded := range []string{"ca.pem", "cert.pem"} { + certFiles[filepath.Join(certsDir, excluded)] = "" + } + + // populates minikube CA + certFiles[filepath.Join(localPath, "ca.crt")] = path.Join(constants.CACertificatesDir, "minikubeCA.pem") + + filtered := map[string]string{} + for k, v := range certFiles { + if v != "" { + filtered[k] = v + } + } + return filtered, nil +} + +func configureCACerts(cmd command.Runner, caCerts map[string]string) error { + getSubjectHash := func(hostpath string) (string, error) { + out, err := cmd.CombinedOutput(fmt.Sprintf("openssl x509 -hash -noout -in '%s'", hostpath)) + if err != nil { + return "", err + } + + stringHash := strings.ReplaceAll(out, "\n", "") + return stringHash, nil + } + + for _, caCertFile := range caCerts { + dstFilename := path.Base(caCertFile) + certStorePath := path.Join(constants.SSLCertStoreDir, dstFilename) + if err := cmd.Run(fmt.Sprintf("sudo test -f '%s'", certStorePath)); err != nil { + if err := cmd.Run(fmt.Sprintf("sudo ln -s '%s' '%s'", caCertFile, certStorePath)); err != nil { + return errors.Wrapf(err, "error making symbol link for certificate %s", caCertFile) + } + } + subjectHash, err := getSubjectHash(caCertFile) + if err != nil { + return errors.Wrapf(err, "error calculating subject hash for certificate %s", caCertFile) + } + subjectHashLink := path.Join(constants.SSLCertStoreDir, fmt.Sprintf("%s.0", subjectHash)) + if err := cmd.Run(fmt.Sprintf("sudo test -f '%s'", subjectHashLink)); err != nil { + if err := cmd.Run(fmt.Sprintf("sudo ln -s '%s' '%s'", certStorePath, subjectHashLink)); err != nil { + return errors.Wrapf(err, "error making subject hash symbol link for certificate %s", caCertFile) + } + } + } + + return nil +} diff --git a/pkg/minikube/bootstrapper/certs_test.go b/pkg/minikube/bootstrapper/certs_test.go index 893c7a083240..209388fe6742 100644 --- a/pkg/minikube/bootstrapper/certs_test.go +++ b/pkg/minikube/bootstrapper/certs_test.go @@ -17,7 +17,9 @@ limitations under the License. package bootstrapper import ( + "fmt" "os" + "path" "path/filepath" "testing" @@ -39,10 +41,37 @@ func TestSetupCerts(t *testing.T) { ServiceCIDR: util.DefaultServiceCIDR, } + if err := os.Mkdir(filepath.Join(tempDir, "certs"), 0777); err != nil { + t.Fatalf("error create certificate directory: %v", err) + } + + if err := util.GenerateCACert( + filepath.Join(tempDir, "certs", "mycert.pem"), + filepath.Join(tempDir, "certs", "mykey.pem"), + "Test Certificate", + ); err != nil { + t.Fatalf("error generating certificate: %v", err) + } + + cmdMap := map[string]string{} + certFilenames := map[string]string{"ca.crt": "minikubeCA.pem", "mycert.pem": "mycert.pem"} + for _, dst := range certFilenames { + certFile := path.Join(constants.CACertificatesDir, dst) + certStorePath := path.Join(constants.SSLCertStoreDir, dst) + certNameHash := "abcdef" + remoteCertHashLink := path.Join(constants.SSLCertStoreDir, fmt.Sprintf("%s.0", certNameHash)) + cmdMap[fmt.Sprintf("sudo ln -s '%s' '%s'", certFile, certStorePath)] = "1" + cmdMap[fmt.Sprintf("openssl x509 -hash -noout -in '%s'", certFile)] = certNameHash + cmdMap[fmt.Sprintf("sudo ln -s '%s' '%s'", certStorePath, remoteCertHashLink)] = "1" + } + f.SetCommandToOutput(cmdMap) + var filesToBeTransferred []string for _, cert := range certs { filesToBeTransferred = append(filesToBeTransferred, filepath.Join(constants.GetMinipath(), cert)) } + filesToBeTransferred = append(filesToBeTransferred, filepath.Join(constants.GetMinipath(), "ca.crt")) + filesToBeTransferred = append(filesToBeTransferred, filepath.Join(constants.GetMinipath(), "certs", "mycert.pem")) if err := SetupCerts(f, k8s); err != nil { t.Fatalf("Error starting cluster: %v", err) diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 435527631005..2045893441a6 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -426,3 +426,8 @@ const ( // DriverDocumentation the documentation of the KVM driver DriverDocumentation = "https://github.com/kubernetes/minikube/blob/master/docs/drivers.md" ) + +const ( + CACertificatesDir = "/usr/share/ca-certificates" + SSLCertStoreDir = "/etc/ssl/certs" +) From 49a91fe7acb549999a62af6bdd7f3013f8a3c22a Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Wed, 7 Aug 2019 20:47:12 +0800 Subject: [PATCH 3/6] Do not fail start command when openssl does not exist Signed-off-by: Zhongcheng Lao --- pkg/minikube/bootstrapper/certs.go | 31 ++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go index 10d26faae719..69be5ef652c0 100644 --- a/pkg/minikube/bootstrapper/certs.go +++ b/pkg/minikube/bootstrapper/certs.go @@ -248,6 +248,10 @@ func collectCACerts() (map[string]string, error) { certsDir := filepath.Join(localPath, "certs") err := filepath.Walk(certsDir, func(hostpath string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info != nil && !info.IsDir() { ext := strings.ToLower(filepath.Ext(hostpath)) if ext == ".crt" || ext == ".pem" { @@ -295,6 +299,15 @@ func configureCACerts(cmd command.Runner, caCerts map[string]string) error { return stringHash, nil } + hasSSLBinary := true + if err := cmd.Run("which openssl"); err != nil { + hasSSLBinary = false + } + + if !hasSSLBinary && len(caCerts) > 0 { + glog.Warning("OpenSSL not found. Please recreate the cluster with the latest minikube ISO.") + } + for _, caCertFile := range caCerts { dstFilename := path.Base(caCertFile) certStorePath := path.Join(constants.SSLCertStoreDir, dstFilename) @@ -303,14 +316,16 @@ func configureCACerts(cmd command.Runner, caCerts map[string]string) error { return errors.Wrapf(err, "error making symbol link for certificate %s", caCertFile) } } - subjectHash, err := getSubjectHash(caCertFile) - if err != nil { - return errors.Wrapf(err, "error calculating subject hash for certificate %s", caCertFile) - } - subjectHashLink := path.Join(constants.SSLCertStoreDir, fmt.Sprintf("%s.0", subjectHash)) - if err := cmd.Run(fmt.Sprintf("sudo test -f '%s'", subjectHashLink)); err != nil { - if err := cmd.Run(fmt.Sprintf("sudo ln -s '%s' '%s'", certStorePath, subjectHashLink)); err != nil { - return errors.Wrapf(err, "error making subject hash symbol link for certificate %s", caCertFile) + if hasSSLBinary { + subjectHash, err := getSubjectHash(caCertFile) + if err != nil { + return errors.Wrapf(err, "error calculating subject hash for certificate %s", caCertFile) + } + subjectHashLink := path.Join(constants.SSLCertStoreDir, fmt.Sprintf("%s.0", subjectHash)) + if err := cmd.Run(fmt.Sprintf("sudo test -f '%s'", subjectHashLink)); err != nil { + if err := cmd.Run(fmt.Sprintf("sudo ln -s '%s' '%s'", certStorePath, subjectHashLink)); err != nil { + return errors.Wrapf(err, "error making subject hash symbol link for certificate %s", caCertFile) + } } } } From 63fdde280f441a4cd907f92b38581c5ace119c4a Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Fri, 9 Aug 2019 23:29:55 +0800 Subject: [PATCH 4/6] Move nested functions out Signed-off-by: Zhongcheng Lao --- pkg/minikube/bootstrapper/certs.go | 78 ++++++++++++++++------------- pkg/minikube/constants/constants.go | 5 -- 2 files changed, 44 insertions(+), 39 deletions(-) diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go index 69be5ef652c0..6af505b505ee 100644 --- a/pkg/minikube/bootstrapper/certs.go +++ b/pkg/minikube/bootstrapper/certs.go @@ -38,6 +38,11 @@ import ( "k8s.io/minikube/pkg/util" ) +const ( + CACertificatesDir = "/usr/share/ca-certificates" + SSLCertStoreDir = "/etc/ssl/certs" +) + var ( certs = []string{ "ca.crt", "ca.key", "apiserver.crt", "apiserver.key", "proxy-client-ca.crt", @@ -219,31 +224,33 @@ func generateCerts(k8s config.KubernetesConfig) error { return nil } -func collectCACerts() (map[string]string, error) { - localPath := constants.GetMinipath() +// isValidPEMCertificate checks whether the input file is a valid PEM certificate (with at least one CERTIFICATE block) +func isValidPEMCertificate(filePath string) (bool, error) { + fileBytes, err := ioutil.ReadFile(filePath) + if err != nil { + return false, err + } - isValidPem := func(hostpath string) (bool, error) { - fileBytes, err := ioutil.ReadFile(hostpath) - if err != nil { - return false, err + for { + block, rest := pem.Decode(fileBytes) + if block == nil { + break } - for { - block, rest := pem.Decode(fileBytes) - if block == nil { - break - } - - if block.Type == "CERTIFICATE" { - // certificate found - return true, nil - } - fileBytes = rest + if block.Type == "CERTIFICATE" { + // certificate found + return true, nil } - - return false, nil + fileBytes = rest } + return false, nil +} + +// collectCACerts looks up all PEM certificates with .crt or .pem extension in ~/.minikube/certs to copy to the host. +// Minikube root CA is also included but libmachine certificates (ca.pem/cert.pem) are excluded. +func collectCACerts() (map[string]string, error) { + localPath := constants.GetMinipath() certFiles := map[string]string{} certsDir := filepath.Join(localPath, "certs") @@ -255,14 +262,14 @@ func collectCACerts() (map[string]string, error) { if info != nil && !info.IsDir() { ext := strings.ToLower(filepath.Ext(hostpath)) if ext == ".crt" || ext == ".pem" { - validPem, err := isValidPem(hostpath) + validPem, err := isValidPEMCertificate(hostpath) if err != nil { return err } if validPem { filename := filepath.Base(hostpath) dst := fmt.Sprintf("%s.%s", strings.TrimSuffix(filename, ext), "pem") - certFiles[hostpath] = path.Join(constants.CACertificatesDir, dst) + certFiles[hostpath] = path.Join(CACertificatesDir, dst) } } } @@ -277,7 +284,7 @@ func collectCACerts() (map[string]string, error) { } // populates minikube CA - certFiles[filepath.Join(localPath, "ca.crt")] = path.Join(constants.CACertificatesDir, "minikubeCA.pem") + certFiles[filepath.Join(localPath, "ca.crt")] = path.Join(CACertificatesDir, "minikubeCA.pem") filtered := map[string]string{} for k, v := range certFiles { @@ -288,17 +295,20 @@ func collectCACerts() (map[string]string, error) { return filtered, nil } -func configureCACerts(cmd command.Runner, caCerts map[string]string) error { - getSubjectHash := func(hostpath string) (string, error) { - out, err := cmd.CombinedOutput(fmt.Sprintf("openssl x509 -hash -noout -in '%s'", hostpath)) - if err != nil { - return "", err - } - - stringHash := strings.ReplaceAll(out, "\n", "") - return stringHash, nil +// getSubjectHash calculates Certificate Subject Hash for creating certificate symlinks +func getSubjectHash(cmd command.Runner, filePath string) (string, error) { + out, err := cmd.CombinedOutput(fmt.Sprintf("openssl x509 -hash -noout -in '%s'", filePath)) + if err != nil { + return "", err } + stringHash := strings.TrimSpace(out) + return stringHash, nil +} + +// configureCACerts looks up and installs all uploaded PEM certificates in /usr/share/ca-certificates to system-wide certificate store (/etc/ssl/certs). +// OpenSSL binary required in minikube ISO +func configureCACerts(cmd command.Runner, caCerts map[string]string) error { hasSSLBinary := true if err := cmd.Run("which openssl"); err != nil { hasSSLBinary = false @@ -310,18 +320,18 @@ func configureCACerts(cmd command.Runner, caCerts map[string]string) error { for _, caCertFile := range caCerts { dstFilename := path.Base(caCertFile) - certStorePath := path.Join(constants.SSLCertStoreDir, dstFilename) + certStorePath := path.Join(SSLCertStoreDir, dstFilename) if err := cmd.Run(fmt.Sprintf("sudo test -f '%s'", certStorePath)); err != nil { if err := cmd.Run(fmt.Sprintf("sudo ln -s '%s' '%s'", caCertFile, certStorePath)); err != nil { return errors.Wrapf(err, "error making symbol link for certificate %s", caCertFile) } } if hasSSLBinary { - subjectHash, err := getSubjectHash(caCertFile) + subjectHash, err := getSubjectHash(cmd, caCertFile) if err != nil { return errors.Wrapf(err, "error calculating subject hash for certificate %s", caCertFile) } - subjectHashLink := path.Join(constants.SSLCertStoreDir, fmt.Sprintf("%s.0", subjectHash)) + subjectHashLink := path.Join(SSLCertStoreDir, fmt.Sprintf("%s.0", subjectHash)) if err := cmd.Run(fmt.Sprintf("sudo test -f '%s'", subjectHashLink)); err != nil { if err := cmd.Run(fmt.Sprintf("sudo ln -s '%s' '%s'", certStorePath, subjectHashLink)); err != nil { return errors.Wrapf(err, "error making subject hash symbol link for certificate %s", caCertFile) diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 2045893441a6..435527631005 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -426,8 +426,3 @@ const ( // DriverDocumentation the documentation of the KVM driver DriverDocumentation = "https://github.com/kubernetes/minikube/blob/master/docs/drivers.md" ) - -const ( - CACertificatesDir = "/usr/share/ca-certificates" - SSLCertStoreDir = "/etc/ssl/certs" -) From 253d658ad31f96d7e51a50f15a4ab58716597bc2 Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Sat, 10 Aug 2019 00:21:47 +0800 Subject: [PATCH 5/6] Fixes test case failure due to moving constants Signed-off-by: Zhongcheng Lao --- pkg/minikube/bootstrapper/certs_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/minikube/bootstrapper/certs_test.go b/pkg/minikube/bootstrapper/certs_test.go index 209388fe6742..d3017ceca4e0 100644 --- a/pkg/minikube/bootstrapper/certs_test.go +++ b/pkg/minikube/bootstrapper/certs_test.go @@ -56,10 +56,10 @@ func TestSetupCerts(t *testing.T) { cmdMap := map[string]string{} certFilenames := map[string]string{"ca.crt": "minikubeCA.pem", "mycert.pem": "mycert.pem"} for _, dst := range certFilenames { - certFile := path.Join(constants.CACertificatesDir, dst) - certStorePath := path.Join(constants.SSLCertStoreDir, dst) + certFile := path.Join(CACertificatesDir, dst) + certStorePath := path.Join(SSLCertStoreDir, dst) certNameHash := "abcdef" - remoteCertHashLink := path.Join(constants.SSLCertStoreDir, fmt.Sprintf("%s.0", certNameHash)) + remoteCertHashLink := path.Join(SSLCertStoreDir, fmt.Sprintf("%s.0", certNameHash)) cmdMap[fmt.Sprintf("sudo ln -s '%s' '%s'", certFile, certStorePath)] = "1" cmdMap[fmt.Sprintf("openssl x509 -hash -noout -in '%s'", certFile)] = certNameHash cmdMap[fmt.Sprintf("sudo ln -s '%s' '%s'", certStorePath, remoteCertHashLink)] = "1" From e3296393a62cd9d58d0372d48f26bcd01f689fdb Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Thu, 15 Aug 2019 20:38:23 +0800 Subject: [PATCH 6/6] Adds tutorial doc for untrusted root CA Signed-off-by: Zhongcheng Lao --- .../Tutorials/untrusted_root_certificate.md | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 site/content/en/docs/Tutorials/untrusted_root_certificate.md diff --git a/site/content/en/docs/Tutorials/untrusted_root_certificate.md b/site/content/en/docs/Tutorials/untrusted_root_certificate.md new file mode 100644 index 000000000000..77093d76f3ef --- /dev/null +++ b/site/content/en/docs/Tutorials/untrusted_root_certificate.md @@ -0,0 +1,42 @@ +--- +title: "Untrusted Root Certificate" +linkTitle: "Untrusted Root Certificate" +weight: 1 +date: 2019-08-15 +description: > + Using minikube with Untrusted Root Certificate +--- + +## Overview + +Most organizations deploy their own Root Certificate and CA service inside the corporate networks. +Internal websites, image repositories and other resources may install SSL server certificates issued by this CA service for security and privacy concerns. +You may install the Root Certificate into the minikube VM to access these corporate resources within the cluster. + +## Prerequisites + +- Corporate X.509 Root Certificate +- Latest minikube binary and ISO + +## Tutorial + +* The certificate must be in PEM format. You may use `openssl` to convert from DER format. + +``` +openssl x509 -inform der -in my_company.cer -out my_company.pem +``` + +* You may need to delete existing minikube VM + +```shell +minikube delete +``` + +* Copy the certificate before creating the minikube VM + +```shell +mkdir -p $HOME/.minikube/certs +cp my_company.pem $HOME/.minikube/certs/my_company.pem + +minikube start +```