kola: add RequiresInternetAccess register flag

Tests with this flag set will not be run when running on a platform that does not provide the networking setup required for Internet. Currently, the only platform where this is the case is `"qemu"`. This avoids needing to manually exclude plaforms using `ExcludePlatforms:` for this case. Fixes: #572
coreos · Jan 22, 2019 · 40e7287 · 40e7287
1 parent c9d73aa
commit 40e7287
Show file tree

Hide file tree

Showing 19 changed files with 118 additions and 93 deletions.
diff --git a/kola/harness.go b/kola/harness.go
@@ -192,6 +192,20 @@ func filterTests(tests map[string]*register.Test, pattern, platform string, vers
 			continue
 		}
 
+		existsIn := func(item string, entries []string) bool {
+			for _, i := range entries {
+				if i == item {
+					return true
+				}
+			}
+			return false
+		}
+
+		if existsIn(platform, register.PlatformsNoInternet) && t.HasFlag(register.RequiresInternetAccess) {
+			plog.Debugf("skipping test %s: Internet required but not supported by platform %s", t.Name, platform)
+			continue
+		}
+
 		isAllowed := func(item string, include, exclude []string) bool {
 			allowed := true
 			for _, i := range include {

diff --git a/kola/register/register.go b/kola/register/register.go
@@ -26,10 +26,18 @@ import (
 type Flag int
 
 const (
-	NoSSHKeyInUserData    Flag = iota // don't inject SSH key into Ignition/cloud-config
-	NoSSHKeyInMetadata                // don't add SSH key to platform metadata
-	NoEmergencyShellCheck             // don't check console output for emergency shell invocation
-	NoEnableSelinux                   // don't enable selinux when starting or rebooting a machine
+	NoSSHKeyInUserData     Flag = iota // don't inject SSH key into Ignition/cloud-config
+	NoSSHKeyInMetadata                 // don't add SSH key to platform metadata
+	NoEmergencyShellCheck              // don't check console output for emergency shell invocation
+	NoEnableSelinux                    // don't enable selinux when starting or rebooting a machine
+	RequiresInternetAccess             // run the test only if the platform supports Internet access
+)
+
+var (
+	// platforms that have no Internet access
+	PlatformsNoInternet = []string{
+		"qemu",
+	}
 )
 
 // Test provides the main test abstraction for kola. The run function is

diff --git a/kola/tests/coretest/core.go b/kola/tests/coretest/core.go
@@ -74,10 +74,10 @@ func init() {
 
 	// tests requiring network connection to internet
 	register.Register(&register.Test{
-		Name:             "cl.internet",
-		Run:              InternetTests,
-		ClusterSize:      1,
-		ExcludePlatforms: []string{"qemu"},
+		Name:        "cl.internet",
+		Run:         InternetTests,
+		ClusterSize: 1,
+		Flags:       []register.Flag{register.RequiresInternetAccess},
 		NativeFuncs: map[string]func() error{
 			"UpdateEngine": TestUpdateEngine,
 			"DockerPing":   TestDockerPing,

diff --git a/kola/tests/crio/crio.go b/kola/tests/crio/crio.go
@@ -147,18 +147,19 @@ var crioContainerTemplate = `{
 // init runs when the package is imported and takes care of registering tests
 func init() {
 	register.Register(&register.Test{
-		Run:              crioBaseTests,
-		ClusterSize:      1,
-		Name:             `crio.base`,
-		ExcludePlatforms: []string{"qemu"}, // crio pods require fetching a kubernetes pause image
-		Distros:          []string{"rhcos"},
+		Run:         crioBaseTests,
+		ClusterSize: 1,
+		Name:        `crio.base`,
+		// crio pods require fetching a kubernetes pause image
+		Flags:   []register.Flag{register.RequiresInternetAccess},
+		Distros: []string{"rhcos"},
 	})
 	register.Register(&register.Test{
-		Run:              crioNetwork,
-		ClusterSize:      2,
-		Name:             "crio.network",
-		ExcludePlatforms: []string{"qemu"},
-		Distros:          []string{"rhcos"},
+		Run:         crioNetwork,
+		ClusterSize: 2,
+		Name:        "crio.network",
+		Flags:       []register.Flag{register.RequiresInternetAccess},
+		Distros:     []string{"rhcos"},
 	})
 }
 

diff --git a/kola/tests/docker/torcx_manifest_pkgs.go b/kola/tests/docker/torcx_manifest_pkgs.go
@@ -33,9 +33,9 @@ func init() {
 		Run:         dockerTorcxManifestPkgs,
 		ClusterSize: 0,
 		Name:        "docker.torcx-manifest-pkgs",
-		// Downloads torcx packages
+		Flags:       []register.Flag{register.RequiresInternetAccess}, // Downloads torcx packages
 		// https://github.com/coreos/bugs/issues/2205 for DO
-		ExcludePlatforms: []string{"qemu", "do"},
+		ExcludePlatforms: []string{"do"},
 		Distros:          []string{"cl"},
 	})
 }

diff --git a/kola/tests/etcd/discovery.go b/kola/tests/etcd/discovery.go
@@ -37,8 +37,8 @@ func init() {
   listen_peer_urls:            http://{PRIVATE_IPV4}:2380
   initial_advertise_peer_urls: http://{PRIVATE_IPV4}:2380
   discovery:                   $discovery`),
-		ExcludePlatforms: []string{"qemu"}, // etcd-member requires networking
-		Distros:          []string{"cl"},
+		Flags:   []register.Flag{register.RequiresInternetAccess}, // etcd-member requires networking
+		Distros: []string{"cl"},
 	})
 
 	register.Register(&register.Test{
@@ -54,7 +54,8 @@ etcd:
   initial_advertise_peer_urls: http://{PRIVATE_IPV4}:2380
   discovery:                   $discovery
 `),
-		ExcludePlatforms: []string{"qemu", "esx"}, // etcd-member requires networking and ct rendering
+		Flags:            []register.Flag{register.RequiresInternetAccess}, // etcd-member requires networking
+		ExcludePlatforms: []string{"esx"},                                  // etcd-member requires ct rendering
 		Distros:          []string{"cl"},
 	})
 
@@ -72,8 +73,8 @@ etcd:
   listen_peer_urls:            http://0.0.0.0:2380
   initial_advertise_peer_urls: http://127.0.0.1:2380
 `),
-		ExcludePlatforms: []string{"qemu"}, // networking to download etcd image
-		Distros:          []string{"cl"},
+		Flags:   []register.Flag{register.RequiresInternetAccess}, // networking to download etcd image
+		Distros: []string{"cl"},
 	})
 }
 
@@ -159,7 +160,7 @@ EOF
 `)
 }
 
-// etcdmemberEtcdctlV3 tests the basic operatoin of the ETCDCTL_API=3 behavior
+// etcdmemberEtcdctlV3 tests the basic operation of the ETCDCTL_API=3 behavior
 // of the etcdctl we ship.
 func etcdmemberEtcdctlV3(c cluster.TestCluster) {
 	m := c.Machines()[0]

diff --git a/kola/tests/etcd/rhcos.go b/kola/tests/etcd/rhcos.go
@@ -47,8 +47,8 @@ func init() {
     ]
   }
 }`),
-		ExcludePlatforms: []string{"qemu"}, // fetching etcd requires networking
-		Distros:          []string{"rhcos"},
+		Flags:   []register.Flag{register.RequiresInternetAccess}, // fetching etcd requires networking
+		Distros: []string{"rhcos"},
 	})
 	register.Register(&register.Test{
 		Run:         rhcosClusterTLS,
@@ -80,8 +80,8 @@ func init() {
     ]
   }
 }`),
-		ExcludePlatforms: []string{"qemu"}, // fetching etcd requires networking
-		Distros:          []string{"rhcos"},
+		Flags:   []register.Flag{register.RequiresInternetAccess}, // fetching etcd requires networking
+		Distros: []string{"rhcos"},
 	})
 }
 

diff --git a/kola/tests/flannel/flannel.go b/kola/tests/flannel/flannel.go
@@ -59,21 +59,21 @@ systemd:
 
 func init() {
 	register.Register(&register.Test{
-		Run:              udp,
-		ClusterSize:      3,
-		Name:             "cl.flannel.udp",
-		ExcludePlatforms: []string{"qemu"},
-		Distros:          []string{"cl"},
-		UserData:         flannelConf.Subst("$type", "udp"),
+		Run:         udp,
+		ClusterSize: 3,
+		Name:        "cl.flannel.udp",
+		Flags:       []register.Flag{register.RequiresInternetAccess}, // requires networking between nodes
+		Distros:     []string{"cl"},
+		UserData:    flannelConf.Subst("$type", "udp"),
 	})
 
 	register.Register(&register.Test{
-		Run:              vxlan,
-		ClusterSize:      3,
-		Name:             "cl.flannel.vxlan",
-		ExcludePlatforms: []string{"qemu"},
-		Distros:          []string{"cl"},
-		UserData:         flannelConf.Subst("$type", "vxlan"),
+		Run:         vxlan,
+		ClusterSize: 3,
+		Name:        "cl.flannel.vxlan",
+		Flags:       []register.Flag{register.RequiresInternetAccess}, // requires networking between nodes
+		Distros:     []string{"cl"},
+		UserData:    flannelConf.Subst("$type", "vxlan"),
 	})
 }
 

diff --git a/kola/tests/ignition/resource.go b/kola/tests/ignition/resource.go
@@ -82,8 +82,9 @@ func init() {
 		Name:        "coreos.ignition.v2_1.resource.remote",
 		Run:         resourceRemote,
 		ClusterSize: 1,
+		Flags:       []register.Flag{register.RequiresInternetAccess},
 		// https://github.com/coreos/bugs/issues/2205 for DO
-		ExcludePlatforms: []string{"qemu", "do"},
+		ExcludePlatforms: []string{"do"},
 		UserData: conf.Ignition(`{
 		  "ignition": {
 		      "version": "2.1.0"
@@ -155,8 +156,9 @@ func init() {
 		Name:        "coreos.ignition.v2_1.resource.s3.versioned",
 		Run:         resourceS3Versioned,
 		ClusterSize: 1,
+		Flags:       []register.Flag{register.RequiresInternetAccess},
 		// https://github.com/coreos/bugs/issues/2205 for DO
-		ExcludePlatforms: []string{"qemu", "do"},
+		ExcludePlatforms: []string{"do"},
 		MinVersion:       semver.Version{Major: 1995},
 		UserData: conf.Ignition(`{
 		  "ignition": {

diff --git a/kola/tests/kubernetes/kubelet_wrapper.go b/kola/tests/kubernetes/kubelet_wrapper.go
@@ -52,8 +52,8 @@ systemd:
       [Install]
       WantedBy=multi-user.target
 `),
-		ExcludePlatforms: []string{"qemu"}, // network access for hyperkube
-		Distros:          []string{"cl"},
+		Flags:   []register.Flag{register.RequiresInternetAccess}, // network access for hyperkube
+		Distros: []string{"cl"},
 	})
 }
 

diff --git a/kola/tests/locksmith/locksmith.go b/kola/tests/locksmith/locksmith.go
@@ -45,8 +45,8 @@ etcd:
   initial_advertise_peer_urls: http://{PRIVATE_IPV4}:2380
   listen_peer_urls:            http://{PRIVATE_IPV4}:2380
   discovery:                   $discovery`),
-		ExcludePlatforms: []string{"qemu"}, // etcd-member requires networking
-		Distros:          []string{"cl"},
+		Flags:   []register.Flag{register.RequiresInternetAccess}, // etcdctl health-check requires networking
+		Distros: []string{"cl"},
 	})
 	register.Register(&register.Test{
 		Name:        "coreos.locksmith.reboot",
@@ -100,8 +100,8 @@ etcd:
     ]
   }
 }`),
-		ExcludePlatforms: []string{"qemu"}, // etcd-member requires networking
-		Distros:          []string{"cl"},
+		Flags:   []register.Flag{register.RequiresInternetAccess}, // Networking required
+		Distros: []string{"cl"},
 	})
 }
 

diff --git a/kola/tests/misc/tls.go b/kola/tests/misc/tls.go
@@ -31,11 +31,11 @@ var (
 
 func init() {
 	register.Register(&register.Test{
-		Run:              TestTLSFetchURLs,
-		ClusterSize:      1,
-		Name:             "coreos.tls.fetch-urls",
-		ExcludePlatforms: []string{"qemu"},  // Networking outside cluster required
-		ExcludeDistros:   []string{"rhcos"}, // wget not included in RHCOS
+		Run:            TestTLSFetchURLs,
+		ClusterSize:    1,
+		Name:           "coreos.tls.fetch-urls",
+		Flags:          []register.Flag{register.RequiresInternetAccess}, // Networking outside cluster required
+		ExcludeDistros: []string{"rhcos"},                                // wget not included in RHCOS
 	})
 }
 

diff --git a/kola/tests/misc/toolbox.go b/kola/tests/misc/toolbox.go
@@ -23,11 +23,11 @@ import (
 
 func init() {
 	register.Register(&register.Test{
-		Run:              dnfInstall,
-		ClusterSize:      1,
-		ExcludePlatforms: []string{"qemu"}, // Network access for toolbox
-		Name:             "cl.toolbox.dnf-install",
-		Distros:          []string{"cl"},
+		Run:         dnfInstall,
+		ClusterSize: 1,
+		Name:        "cl.toolbox.dnf-install",
+		Flags:       []register.Flag{register.RequiresInternetAccess}, // Network access for toolbox
+		Distros:     []string{"cl"},
 	})
 }
 

diff --git a/kola/tests/ostree/basic.go b/kola/tests/ostree/basic.go
@@ -39,12 +39,12 @@ func init() {
 	})
 
 	register.Register(&register.Test{
-		Run:              ostreeRemoteTest,
-		ClusterSize:      1,
-		Name:             "ostree.remote",
-		Distros:          []string{"rhcos", "fcos"},
-		ExcludePlatforms: []string{"qemu"}, // need network to contact remote
-		FailFast:         true,
+		Run:         ostreeRemoteTest,
+		ClusterSize: 1,
+		Name:        "ostree.remote",
+		Distros:     []string{"rhcos", "fcos"},
+		Flags:       []register.Flag{register.RequiresInternetAccess}, // need network to contact remote
+		FailFast:    true,
 	})
 }
 

diff --git a/kola/tests/ostree/unlock.go b/kola/tests/ostree/unlock.go
@@ -25,20 +25,20 @@ import (
 
 func init() {
 	register.Register(&register.Test{
-		Run:              ostreeUnlockTest,
-		ClusterSize:      1,
-		Name:             "ostree.unlock",
-		Distros:          []string{"rhcos", "fcos"},
-		ExcludePlatforms: []string{"qemu"}, // need network to pull RPM
-		FailFast:         true,
+		Run:         ostreeUnlockTest,
+		ClusterSize: 1,
+		Name:        "ostree.unlock",
+		Flags:       []register.Flag{register.RequiresInternetAccess}, // need network to pull RPM
+		Distros:     []string{"rhcos", "fcos"},
+		FailFast:    true,
 	})
 	register.Register(&register.Test{
-		Run:              ostreeHotfixTest,
-		ClusterSize:      1,
-		Name:             "ostree.hotfix",
-		Distros:          []string{"rhcos", "fcos"},
-		ExcludePlatforms: []string{"qemu"}, // need network to pull RPM
-		FailFast:         true,
+		Run:         ostreeHotfixTest,
+		ClusterSize: 1,
+		Flags:       []register.Flag{register.RequiresInternetAccess}, // need network to pull RPM
+		Name:        "ostree.hotfix",
+		Distros:     []string{"rhcos", "fcos"},
+		FailFast:    true,
 	})
 
 }

diff --git a/kola/tests/podman/podman.go b/kola/tests/podman/podman.go
@@ -41,12 +41,12 @@ func init() {
 		Distros:     []string{"rhcos"},
 	})
 	register.Register(&register.Test{
-		Run:              podmanWorkflow,
-		ClusterSize:      1,
-		Name:             `podman.workflow`,
-		ExcludePlatforms: []string{"qemu"}, // Requires internet for pulling nginx
-		Distros:          []string{"rhcos"},
-		FailFast:         true,
+		Run:         podmanWorkflow,
+		ClusterSize: 1,
+		Name:        `podman.workflow`,
+		Flags:       []register.Flag{register.RequiresInternetAccess}, // For pulling nginx
+		Distros:     []string{"rhcos"},
+		FailFast:    true,
 	})
 	register.Register(&register.Test{
 		Run:         podmanNetworkTest,

diff --git a/kola/tests/rkt/rkt.go b/kola/tests/rkt/rkt.go
@@ -41,12 +41,12 @@ var config = conf.Ignition(`{
 
 func init() {
 	register.Register(&register.Test{
-		Run:              rktEtcd,
-		ClusterSize:      1,
-		ExcludePlatforms: []string{"qemu"},
-		Distros:          []string{"cl"},
-		Name:             "cl.rkt.etcd3",
-		UserData:         config,
+		Name:        "cl.rkt.etcd3",
+		Run:         rktEtcd,
+		ClusterSize: 1,
+		Flags:       []register.Flag{register.RequiresInternetAccess}, // etcdctl health-check requires networking
+		Distros:     []string{"cl"},
+		UserData:    config,
 	})
 
 	register.Register(&register.Test{

diff --git a/kola/tests/rpmostree/deployments.go b/kola/tests/rpmostree/deployments.go
@@ -75,8 +75,8 @@ func init() {
   }
 }`),
 
-		Distros:          []string{"rhcos"},
-		ExcludePlatforms: []string{"qemu"}, // these need network to retrieve bits
+		Distros: []string{"rhcos"},
+		Flags:   []register.Flag{register.RequiresInternetAccess}, // these need network to retrieve bits
 	})
 }
 

diff --git a/kola/tests/rpmostree/status.go b/kola/tests/rpmostree/status.go
@@ -109,6 +109,5 @@ func rpmOstreeStatus(c cluster.TestCluster) {
 
 	if rpmOstreeVersion != status.Deployments[0].Version {
 		c.Fatalf(`The version numbers did not match -> from JSON: %q; from stdout: %q`, status.Deployments[0].Version, rpmOstreeVersion)
-
 	}
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -109,6 +109,5 @@ func rpmOstreeStatus(c cluster.TestCluster) { @@
     	if rpmOstreeVersion != status.Deployments[0].Version {
     		c.Fatalf(`The version numbers did not match -> from JSON: %q; from stdout: %q`, status.Deployments[0].Version, rpmOstreeVersion)
     	}
     }