@@ -30,6 +30,7 @@ import (
3030
3131 "github.com/docker/machine/libmachine/state"
3232 "github.com/golang/glog"
33+ core "k8s.io/api/core/v1"
3334 meta "k8s.io/apimachinery/pkg/apis/meta/v1"
3435 "k8s.io/apimachinery/pkg/util/wait"
3536 "k8s.io/client-go/kubernetes"
@@ -79,6 +80,68 @@ func apiServerPID(cr command.Runner) (int, error) {
7980 return strconv .Atoi (s )
8081}
8182
83+ // ExpectedComponentsRunning returns whether or not all expected components are running
84+ func ExpectedComponentsRunning (cs * kubernetes.Clientset ) error {
85+ expected := []string {
86+ "kube-dns" , // coredns
87+ "etcd" ,
88+ "kube-apiserver" ,
89+ "kube-controller-manager" ,
90+ "kube-proxy" ,
91+ "kube-scheduler" ,
92+ }
93+
94+ found := map [string ]bool {}
95+
96+ pods , err := cs .CoreV1 ().Pods ("kube-system" ).List (meta.ListOptions {})
97+ if err != nil {
98+ return err
99+ }
100+
101+ for _ , pod := range pods .Items {
102+ glog .Infof ("found pod: %s" , podStatusMsg (pod ))
103+ if pod .Status .Phase != core .PodRunning {
104+ continue
105+ }
106+ for k , v := range pod .ObjectMeta .Labels {
107+ if k == "component" || k == "k8s-app" {
108+ found [v ] = true
109+ }
110+ }
111+ }
112+
113+ missing := []string {}
114+ for _ , e := range expected {
115+ if ! found [e ] {
116+ missing = append (missing , e )
117+ }
118+ }
119+ if len (missing ) > 0 {
120+ return fmt .Errorf ("missing components: %v" , strings .Join (missing , ", " ))
121+ }
122+ return nil
123+ }
124+
125+ // podStatusMsg returns a human-readable pod status, for generating debug status
126+ func podStatusMsg (pod core.Pod ) string {
127+ var sb strings.Builder
128+ sb .WriteString (fmt .Sprintf ("%q [%s] %s" , pod .ObjectMeta .GetName (), pod .ObjectMeta .GetUID (), pod .Status .Phase ))
129+ for i , c := range pod .Status .Conditions {
130+ if c .Reason != "" {
131+ if i == 0 {
132+ sb .WriteString (": " )
133+ } else {
134+ sb .WriteString (" / " )
135+ }
136+ sb .WriteString (fmt .Sprintf ("%s:%s" , c .Type , c .Reason ))
137+ }
138+ if c .Message != "" {
139+ sb .WriteString (fmt .Sprintf (" (%s)" , c .Message ))
140+ }
141+ }
142+ return sb .String ()
143+ }
144+
82145// WaitForSystemPods verifies essential pods for running kurnetes is running
83146func WaitForSystemPods (r cruntime.Manager , bs bootstrapper.Bootstrapper , cfg config.ClusterConfig , cr command.Runner , client * kubernetes.Clientset , start time.Time , timeout time.Duration ) error {
84147 glog .Info ("waiting for kube-system pods to appear ..." )
@@ -100,6 +163,10 @@ func WaitForSystemPods(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg con
100163 return false , nil
101164 }
102165 glog .Infof ("%d kube-system pods found" , len (pods .Items ))
166+ for _ , pod := range pods .Items {
167+ glog .Infof (podStatusMsg (pod ))
168+ }
169+
103170 if len (pods .Items ) < 2 {
104171 return false , nil
105172 }
@@ -160,7 +227,7 @@ func APIServerStatus(cr command.Runner, ip net.IP, port int) (state.State, error
160227
161228 pid , err := apiServerPID (cr )
162229 if err != nil {
163- glog .Warningf ("unable to get apiserver pid: %v" , err )
230+ glog .Warningf ("stopped: unable to get apiserver pid: %v" , err )
164231 return state .Stopped , nil
165232 }
166233
@@ -206,6 +273,7 @@ func apiServerHealthz(ip net.IP, port int) (state.State, error) {
206273 resp , err := client .Get (url )
207274 // Connection refused, usually.
208275 if err != nil {
276+ glog .Infof ("stopped: %s: %v" , url , err )
209277 return state .Stopped , nil
210278 }
211279 if resp .StatusCode == http .StatusUnauthorized {
0 commit comments