Merge pull request #3464 from ceason/tunnel-dns

WIP: Resolve service FQDNs from host during 'minikube tunnel' (macOS)

Author: tstromberg

pkg/minikube/tunnel/cluster_inspector.go

@@ -26,6 +26,7 @@ import (
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/util"
 )
 
 type clusterInspector struct {
@@ -93,9 +94,14 @@ func getRoute(host *host.Host, clusterConfig config.Config) (*Route, error) {
 	if ip == nil {
 		return nil, fmt.Errorf("invalid IP for host %s", hostDriverIP)
 	}
-
+	dnsIp, err := util.GetDNSIP(ipNet.String())
+	if err != nil {
+		return nil, err
+	}
 	return &Route{
-		Gateway:  ip,
-		DestCIDR: ipNet,
+		Gateway:       ip,
+		DestCIDR:      ipNet,
+		ClusterDomain: clusterConfig.KubernetesConfig.DNSDomain,
+		ClusterDNSIP:  dnsIp,
 	}, nil
 }

pkg/minikube/tunnel/cluster_inspector_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package tunnel
 
 import (
+	"k8s.io/minikube/pkg/util"
 	"testing"
 
 	"net"
@@ -78,10 +79,12 @@ func TestMinikubeCheckReturnsHostInformation(t *testing.T) {
 
 	ip := net.ParseIP("1.2.3.4")
 	_, ipNet, _ := net.ParseCIDR("96.0.0.0/12")
+	dnsIp, _ := util.GetDNSIP(ipNet.String())
 
 	expectedRoute := &Route{
-		Gateway:  ip,
-		DestCIDR: ipNet,
+		Gateway:      ip,
+		DestCIDR:     ipNet,
+		ClusterDNSIP: dnsIp,
 	}
 
 	if s != Running {

pkg/minikube/tunnel/route_darwin.go

@@ -18,7 +18,9 @@ package tunnel
 
 import (
 	"fmt"
+	"io/ioutil"
 	"net"
+	"os"
 	"os/exec"
 	"regexp"
 	"strings"
@@ -34,6 +36,9 @@ func (router *osRouter) EnsureRouteIsAdded(route *Route) error {
 	if exists {
 		return nil
 	}
+	if err := writeResolverFile(route); err != nil {
+		return fmt.Errorf("could not write /etc/resolver/{cluster_domain} file: %s", err)
+	}
 
 	serviceCIDR := route.DestCIDR.String()
 	gatewayIP := route.Gateway.String()
@@ -162,5 +167,37 @@ func (router *osRouter) Cleanup(route *Route) error {
 	if !re.MatchString(message) {
 		return fmt.Errorf("error deleting route: %s, %d", message, len(strings.Split(message, "\n")))
 	}
+	// idempotent removal of cluster domain dns
+	resolverFile := fmt.Sprintf("/etc/resolver/%s", route.ClusterDomain)
+	command = exec.Command("sudo", "rm", "-f", resolverFile)
+	if err := command.Run(); err != nil {
+		return fmt.Errorf("could not remove %s: %s", resolverFile, err)
+	}
+	return nil
+}
+
+func writeResolverFile(route *Route) error {
+	resolverFile := "/etc/resolver/" + route.ClusterDomain
+	content := fmt.Sprintf("nameserver %s\nsearch_order 1\n", route.ClusterDNSIP)
+	// write resolver content into tmpFile, then copy it to /etc/resolver/clusterDomain
+	tmpFile, err := ioutil.TempFile("", "minikube-tunnel-resolver-")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tmpFile.Name())
+	if _, err = tmpFile.WriteString(content); err != nil {
+		return err
+	}
+	if err = tmpFile.Close(); err != nil {
+		return err
+	}
+	command := exec.Command("sudo", "mkdir", "-p", "/etc/resolver")
+	if err := command.Run(); err != nil {
+		return err
+	}
+	command = exec.Command("sudo", "cp", "-f", tmpFile.Name(), resolverFile)
+	if err := command.Run(); err != nil {
+		return err
+	}
 	return nil
 }

pkg/minikube/tunnel/route_darwin_test.go

@@ -35,9 +35,14 @@ func TestDarwinRouteFailsOnConflictIntegrationTest(t *testing.T) {
 			IP:   net.IPv4(10, 96, 0, 0),
 			Mask: net.IPv4Mask(255, 240, 0, 0),
 		},
+		ClusterDomain: "cluster.local",
+		ClusterDNSIP:  net.IPv4(10, 96, 0, 10),
 	}
+	conflictingCfg := *cfg
+	conflictingCfg.Gateway = net.IPv4(127, 0, 0, 2)
 
-	addRoute(t, "10.96.0.0/12", "127.0.0.2")
+	addRoute(t, &conflictingCfg)
+	defer cleanRoute(t, &conflictingCfg)
 	err := (&osRouter{}).EnsureRouteIsAdded(cfg)
 	if err == nil {
 		t.Errorf("add should have error, but it is nil")
@@ -51,9 +56,11 @@ func TestDarwinRouteIdempotentIntegrationTest(t *testing.T) {
 			IP:   net.IPv4(10, 96, 0, 0),
 			Mask: net.IPv4Mask(255, 240, 0, 0),
 		},
+		ClusterDomain: "cluster.local",
+		ClusterDNSIP:  net.IPv4(10, 96, 0, 10),
 	}
 
-	cleanRoute(t, "10.96.0.0/12")
+	cleanRoute(t, cfg)
 	err := (&osRouter{}).EnsureRouteIsAdded(cfg)
 	if err != nil {
 		t.Errorf("add error: %s", err)
@@ -64,7 +71,7 @@ func TestDarwinRouteIdempotentIntegrationTest(t *testing.T) {
 		t.Errorf("add error: %s", err)
 	}
 
-	cleanRoute(t, "10.96.0.0/12")
+	cleanRoute(t, cfg)
 }
 
 func TestDarwinRouteCleanupIdempontentIntegrationTest(t *testing.T) {
@@ -75,10 +82,12 @@ func TestDarwinRouteCleanupIdempontentIntegrationTest(t *testing.T) {
 			IP:   net.IPv4(10, 96, 0, 0),
 			Mask: net.IPv4Mask(255, 240, 0, 0),
 		},
+		ClusterDomain: "cluster.local",
+		ClusterDNSIP:  net.IPv4(10, 96, 0, 10),
 	}
 
-	cleanRoute(t, "10.96.0.0/12")
-	addRoute(t, "10.96.0.0/12", "192.168.1.1")
+	cleanRoute(t, cfg)
+	addRoute(t, cfg)
 	err := (&osRouter{}).Cleanup(cfg)
 	if err != nil {
 		t.Errorf("cleanup failed with %s", err)
@@ -89,20 +98,32 @@ func TestDarwinRouteCleanupIdempontentIntegrationTest(t *testing.T) {
 	}
 }
 
-func addRoute(t *testing.T, cidr string, gw string) {
+func addRoute(t *testing.T, r *Route) {
+	cidr := r.DestCIDR.String()
+	gw := r.Gateway.String()
 	command := exec.Command("sudo", "route", "-n", "add", cidr, gw)
 	_, err := command.CombinedOutput()
 	if err != nil {
 		t.Logf("add route error (should be ok): %s", err)
 	}
+	err = writeResolverFile(r)
+	if err != nil {
+		t.Logf("add route DNS resolver error (should be ok): %s", err)
+	}
 }
 
-func cleanRoute(t *testing.T, cidr string) {
+func cleanRoute(t *testing.T, r *Route) {
+	cidr := r.DestCIDR.String()
 	command := exec.Command("sudo", "route", "-n", "delete", cidr)
 	_, err := command.CombinedOutput()
 	if err != nil {
 		t.Logf("cleanup error (should be ok): %s", err)
 	}
+	command = exec.Command("sudo", "rm", "-f", fmt.Sprintf("/etc/resolver/%s", r.ClusterDomain))
+	_, err = command.CombinedOutput()
+	if err != nil {
+		t.Logf("cleanup DNS resolver error (should be ok): %s", err)
+	}
 }
 
 func TestCIDRPadding(t *testing.T) {

pkg/minikube/tunnel/route_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package tunnel
 
 import (
+	"k8s.io/minikube/pkg/util"
 	"net"
 	"reflect"
 	"testing"
@@ -130,10 +131,12 @@ got
 func unsafeParseRoute(gatewayIP string, destCIDR string) *Route {
 	ip := net.ParseIP(gatewayIP)
 	_, ipNet, _ := net.ParseCIDR(destCIDR)
+	dnsIp, _ := util.GetDNSIP(ipNet.String())
 
 	expectedRoute := &Route{
-		Gateway:  ip,
-		DestCIDR: ipNet,
+		Gateway:      ip,
+		DestCIDR:     ipNet,
+		ClusterDNSIP: dnsIp,
 	}
 	return expectedRoute
 }

pkg/minikube/tunnel/types.go

@@ -61,8 +61,10 @@ func (t *Status) String() string {
 
 // Route represents a route
 type Route struct {
-	Gateway  net.IP
-	DestCIDR *net.IPNet
+	Gateway       net.IP
+	DestCIDR      *net.IPNet
+	ClusterDomain string
+	ClusterDNSIP  net.IP
 }
 
 func (r *Route) String() string {