Skip to content

Commit c3cfedf

Browse files
authored
Merge pull request #5121 from medyagh/refactor_util_kube
Add wait-timeout flag to start command and refactor util/kubernetes
2 parents c5a06fd + dd94e15 commit c3cfedf

22 files changed

+189
-279
lines changed

cmd/minikube/cmd/start.go

+7-4
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ import (
4646
"k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm"
4747
"k8s.io/minikube/pkg/minikube/cluster"
4848
"k8s.io/minikube/pkg/minikube/command"
49+
"k8s.io/minikube/pkg/minikube/config"
4950
cfg "k8s.io/minikube/pkg/minikube/config"
5051
"k8s.io/minikube/pkg/minikube/constants"
5152
"k8s.io/minikube/pkg/minikube/cruntime"
@@ -102,6 +103,7 @@ const (
102103
dnsProxy = "dns-proxy"
103104
hostDNSResolver = "host-dns-resolver"
104105
waitUntilHealthy = "wait"
106+
waitTimeout = "wait-timeout"
105107
)
106108

107109
var (
@@ -111,7 +113,7 @@ var (
111113
insecureRegistry []string
112114
apiServerNames []string
113115
apiServerIPs []net.IP
114-
extraOptions pkgutil.ExtraOptionSlice
116+
extraOptions config.ExtraOptionSlice
115117
)
116118

117119
func init() {
@@ -148,6 +150,7 @@ func initMinikubeFlags() {
148150
startCmd.Flags().String(networkPlugin, "", "The name of the network plugin.")
149151
startCmd.Flags().Bool(enableDefaultCNI, false, "Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \"--network-plugin=cni\".")
150152
startCmd.Flags().Bool(waitUntilHealthy, true, "Wait until Kubernetes core services are healthy before exiting.")
153+
startCmd.Flags().Duration(waitTimeout, 3*time.Minute, "max time to wait per Kubernetes core services to be healthy.")
151154
}
152155

153156
// initKubernetesFlags inits the commandline flags for kubernetes related options
@@ -321,7 +324,7 @@ func runStart(cmd *cobra.Command, args []string) {
321324
// special ops for none driver, like change minikube directory.
322325
prepareNone(viper.GetString(vmDriver))
323326
if viper.GetBool(waitUntilHealthy) {
324-
if err := bs.WaitCluster(config.KubernetesConfig); err != nil {
327+
if err := bs.WaitCluster(config.KubernetesConfig, viper.GetDuration(waitTimeout)); err != nil {
325328
exit.WithError("Wait failed", err)
326329
}
327330
}
@@ -538,8 +541,8 @@ func validateConfig() {
538541

539542
// check that kubeadm extra args contain only whitelisted parameters
540543
for param := range extraOptions.AsMap().Get(kubeadm.Kubeadm) {
541-
if !pkgutil.ContainsString(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmCmdParam], param) &&
542-
!pkgutil.ContainsString(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmConfigParam], param) {
544+
if !cfg.ContainsParam(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmCmdParam], param) &&
545+
!cfg.ContainsParam(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmConfigParam], param) {
543546
exit.UsageT("Sorry, the kubeadm.{{.parameter_name}} parameter is currently not supported by --extra-config", out.V{"parameter_name": param})
544547
}
545548
}

hack/jenkins/common.sh

+5-3
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,11 @@ export MINIKUBE_HOME="${TEST_HOME}/.minikube"
262262
export MINIKUBE_WANTREPORTERRORPROMPT=False
263263
export KUBECONFIG="${TEST_HOME}/kubeconfig"
264264

265+
# Build the gvisor image. This will be copied into minikube and loaded by ctr.
266+
# Used by TestContainerd for Gvisor Test.
267+
docker build -t gcr.io/k8s-minikube/gvisor-addon:latest -f testdata/gvisor-addon-Dockerfile out
268+
269+
265270
# Display the default image URL
266271
echo ""
267272
echo ">> ISO URL"
@@ -295,9 +300,6 @@ ${SUDO_PREFIX} rm -f "${KUBECONFIG}" || true
295300
rmdir "${TEST_HOME}"
296301
echo ">> ${TEST_HOME} completed at $(date)"
297302

298-
# Build the gvisor image. This will be copied into minikube and loaded by ctr.
299-
docker build -t gcr.io/k8s-minikube/gvisor-addon:latest -f testdata/gvisor-addon-Dockerfile out
300-
301303
if [[ "${MINIKUBE_LOCATION}" != "master" ]]; then
302304
readonly target_url="https://storage.googleapis.com/minikube-builds/logs/${MINIKUBE_LOCATION}/${JOB_NAME}.txt"
303305
curl -s "https://api.github.com/repos/kubernetes/minikube/statuses/${COMMIT}?access_token=$access_token" \

pkg/util/kubernetes.go pkg/kapi/kapi.go

+19-125
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
1414
limitations under the License.
1515
*/
1616

17-
package util
17+
package kapi
1818

1919
import (
2020
"context"
@@ -29,15 +29,13 @@ import (
2929
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
3030
"k8s.io/apimachinery/pkg/fields"
3131
"k8s.io/apimachinery/pkg/labels"
32-
"k8s.io/apimachinery/pkg/runtime"
3332
"k8s.io/apimachinery/pkg/runtime/schema"
3433
"k8s.io/apimachinery/pkg/util/wait"
3534
"k8s.io/apimachinery/pkg/watch"
3635
"k8s.io/client-go/kubernetes"
37-
"k8s.io/client-go/tools/cache"
3836
"k8s.io/client-go/tools/clientcmd"
3937
watchtools "k8s.io/client-go/tools/watch"
40-
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
38+
kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
4139
"k8s.io/minikube/pkg/minikube/proxy"
4240
)
4341

@@ -48,30 +46,8 @@ var (
4846
ReasonableStartTime = time.Minute * 5
4947
)
5048

51-
// PodStore stores pods
52-
type PodStore struct {
53-
cache.Store
54-
stopCh chan struct{}
55-
Reflector *cache.Reflector
56-
}
57-
58-
// List lists the pods
59-
func (s *PodStore) List() []*core.Pod {
60-
objects := s.Store.List()
61-
pods := make([]*core.Pod, 0)
62-
for _, o := range objects {
63-
pods = append(pods, o.(*core.Pod))
64-
}
65-
return pods
66-
}
67-
68-
// Stop stops the pods
69-
func (s *PodStore) Stop() {
70-
close(s.stopCh)
71-
}
72-
73-
// GetClient gets the client from config
74-
func GetClient(kubectlContext ...string) (kubernetes.Interface, error) {
49+
// Client gets the kuberentes client from default kubeconfig
50+
func Client(kubectlContext ...string) (kubernetes.Interface, error) {
7551
loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
7652
configOverrides := &clientcmd.ConfigOverrides{}
7753
if kubectlContext != nil {
@@ -92,50 +68,16 @@ func GetClient(kubectlContext ...string) (kubernetes.Interface, error) {
9268
return client, nil
9369
}
9470

95-
// NewPodStore creates a new PodStore
96-
func NewPodStore(c kubernetes.Interface, namespace string, label fmt.Stringer, field fmt.Stringer) *PodStore {
97-
lw := &cache.ListWatch{
98-
ListFunc: func(options meta.ListOptions) (runtime.Object, error) {
99-
options.LabelSelector = label.String()
100-
options.FieldSelector = field.String()
101-
obj, err := c.CoreV1().Pods(namespace).List(options)
102-
return runtime.Object(obj), err
103-
},
104-
WatchFunc: func(options meta.ListOptions) (watch.Interface, error) {
105-
options.LabelSelector = label.String()
106-
options.FieldSelector = field.String()
107-
return c.CoreV1().Pods(namespace).Watch(options)
108-
},
109-
}
110-
store := cache.NewStore(cache.MetaNamespaceKeyFunc)
111-
stopCh := make(chan struct{})
112-
reflector := cache.NewReflector(lw, &core.Pod{}, store, 0)
113-
go reflector.Run(stopCh)
114-
return &PodStore{Store: store, stopCh: stopCh, Reflector: reflector}
115-
}
116-
117-
// StartPods starts all pods
118-
func StartPods(c kubernetes.Interface, namespace string, pod core.Pod, waitForRunning bool) error {
119-
pod.ObjectMeta.Labels["name"] = pod.Name
120-
if waitForRunning {
121-
label := labels.SelectorFromSet(labels.Set(map[string]string{"name": pod.Name}))
122-
err := WaitForPodsWithLabelRunning(c, namespace, label)
123-
if err != nil {
124-
return fmt.Errorf("error waiting for pod %s to be running: %v", pod.Name, err)
125-
}
126-
}
127-
return nil
128-
}
129-
13071
// WaitForPodsWithLabelRunning waits for all matching pods to become Running and at least one matching pod exists.
131-
func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels.Selector) error {
72+
func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels.Selector, timeOut ...time.Duration) error {
73+
start := time.Now()
13274
glog.Infof("Waiting for pod with label %q in ns %q ...", ns, label)
13375
lastKnownPodNumber := -1
134-
return wait.PollImmediate(constants.APICallRetryInterval, ReasonableStartTime, func() (bool, error) {
76+
f := func() (bool, error) {
13577
listOpts := meta.ListOptions{LabelSelector: label.String()}
13678
pods, err := c.CoreV1().Pods(ns).List(listOpts)
13779
if err != nil {
138-
glog.Infof("error getting Pods with label selector %q [%v]\n", label.String(), err)
80+
glog.Infof("temporary error: getting Pods with label selector %q : [%v]\n", label.String(), err)
13981
return false, nil
14082
}
14183

@@ -150,45 +92,23 @@ func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels
15092

15193
for _, pod := range pods.Items {
15294
if pod.Status.Phase != core.PodRunning {
95+
glog.Infof("waiting for pod %q, current state: %s: [%v]\n", label.String(), pod.Status.Phase, err)
15396
return false, nil
15497
}
15598
}
15699

157100
return true, nil
158-
})
159-
}
160-
161-
// WaitForPodDelete waits for a pod to be deleted
162-
func WaitForPodDelete(c kubernetes.Interface, ns string, label fmt.Stringer) error {
163-
return wait.PollImmediate(constants.APICallRetryInterval, ReasonableMutateTime, func() (bool, error) {
164-
listOpts := meta.ListOptions{LabelSelector: label.String()}
165-
pods, err := c.CoreV1().Pods(ns).List(listOpts)
166-
if err != nil {
167-
glog.Infof("error getting Pods with label selector %q [%v]\n", label.String(), err)
168-
return false, nil
169-
}
170-
return len(pods.Items) == 0, nil
171-
})
172-
}
173-
174-
// WaitForEvent waits for the given event to appear
175-
func WaitForEvent(c kubernetes.Interface, ns string, reason string) error {
176-
return wait.PollImmediate(constants.APICallRetryInterval, ReasonableMutateTime, func() (bool, error) {
177-
events, err := c.EventsV1beta1().Events("default").List(meta.ListOptions{})
178-
if err != nil {
179-
glog.Infof("error getting events: %v", err)
180-
return false, nil
181-
}
182-
for _, e := range events.Items {
183-
if e.Reason == reason {
184-
return true, nil
185-
}
186-
}
187-
return false, nil
188-
})
101+
}
102+
t := ReasonableStartTime
103+
if timeOut != nil {
104+
t = timeOut[0]
105+
}
106+
err := wait.PollImmediate(kconst.APICallRetryInterval, t, f)
107+
glog.Infof("duration metric: took %s to wait for %s ...", time.Since(start), label)
108+
return err
189109
}
190110

191-
// WaitForRCToStabilize waits till the RC has a matching generation/replica count between spec and status.
111+
// WaitForRCToStabilize waits till the RC has a matching generation/replica count between spec and status. used by integration tests
192112
func WaitForRCToStabilize(c kubernetes.Interface, ns, name string, timeout time.Duration) error {
193113
options := meta.ListOptions{FieldSelector: fields.Set{
194114
"metadata.name": name,
@@ -222,7 +142,7 @@ func WaitForRCToStabilize(c kubernetes.Interface, ns, name string, timeout time.
222142
return err
223143
}
224144

225-
// WaitForDeploymentToStabilize waits till the Deployment has a matching generation/replica count between spec and status.
145+
// WaitForDeploymentToStabilize waits till the Deployment has a matching generation/replica count between spec and status. used by integrationt tests
226146
func WaitForDeploymentToStabilize(c kubernetes.Interface, ns, name string, timeout time.Duration) error {
227147
options := meta.ListOptions{FieldSelector: fields.Set{
228148
"metadata.name": name,
@@ -281,32 +201,6 @@ func WaitForService(c kubernetes.Interface, namespace, name string, exist bool,
281201
return nil
282202
}
283203

284-
// WaitForServiceEndpointsNum waits until the amount of endpoints that implement service to expectNum.
285-
func WaitForServiceEndpointsNum(c kubernetes.Interface, namespace, serviceName string, expectNum int, interval, timeout time.Duration) error {
286-
return wait.Poll(interval, timeout, func() (bool, error) {
287-
glog.Infof("Waiting for amount of service:%s endpoints to be %d", serviceName, expectNum)
288-
list, err := c.CoreV1().Endpoints(namespace).List(meta.ListOptions{})
289-
if err != nil {
290-
return false, err
291-
}
292-
293-
for _, e := range list.Items {
294-
if e.Name == serviceName && countEndpointsNum(&e) == expectNum {
295-
return true, nil
296-
}
297-
}
298-
return false, nil
299-
})
300-
}
301-
302-
func countEndpointsNum(e *core.Endpoints) int {
303-
num := 0
304-
for _, sub := range e.Subsets {
305-
num += len(sub.Addresses)
306-
}
307-
return num
308-
}
309-
310204
// IsRetryableAPIError returns if this error is retryable or not
311205
func IsRetryableAPIError(err error) bool {
312206
return apierr.IsTimeout(err) || apierr.IsServerTimeout(err) || apierr.IsTooManyRequests(err) || apierr.IsInternalError(err)

pkg/minikube/bootstrapper/bootstrapper.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package bootstrapper
1818

1919
import (
2020
"net"
21+
"time"
2122

2223
"k8s.io/minikube/pkg/minikube/config"
2324
"k8s.io/minikube/pkg/minikube/constants"
@@ -39,7 +40,7 @@ type Bootstrapper interface {
3940
UpdateCluster(config.KubernetesConfig) error
4041
RestartCluster(config.KubernetesConfig) error
4142
DeleteCluster(config.KubernetesConfig) error
42-
WaitCluster(config.KubernetesConfig) error
43+
WaitCluster(config.KubernetesConfig, time.Duration) error
4344
// LogCommands returns a map of log type to a command which will display that log.
4445
LogCommands(LogOptions) map[string]string
4546
SetupCerts(cfg config.KubernetesConfig) error

0 commit comments

Comments
 (0)