[prezha@codex minikube (ha)]$ make integration -e TEST_ARGS="-minikube-start-args='--kubernetes-version=v1.29.0 --driver=docker --container-runtime=docker --alsologtostderr -v=7' -test.run TestHA --cleanup=false" go test -ldflags="-X k8s.io/minikube/pkg/version.version=v1.32.0 -X k8s.io/minikube/pkg/version.isoVersion=v1.32.1-1702708929-17806 -X k8s.io/minikube/pkg/version.gitCommitID="6731cdee84963068917e40d819e088c6d59d5516-dirty" -X k8s.io/minikube/pkg/version.storageProvisionerVersion=v5" -v -test.timeout=90m ./test/integration --tags="integration " -minikube-start-args='--kubernetes-version=v1.29.0 --driver=docker --container-runtime=docker --alsologtostderr -v=7' -test.run TestHA --cleanup=false 2>&1 | tee "./out/testout_6731cdee8.txt" Found 16 cores, limiting parallelism with --test.parallel=9 === RUN TestHA ha_test.go:45: (dbg) Run: docker version -f {{.Server.Version}} === RUN TestHA/serial === RUN TestHA/serial/StartCluster ha_test.go:101: (dbg) Run: out/minikube start -p ha-644161 --wait=true --memory=2200 --ha -v=7 --alsologtostderr --kubernetes-version=v1.29.0 --driver=docker --container-runtime=docker --alsologtostderr -v=7 ha_test.go:101: (dbg) Done: out/minikube start -p ha-644161 --wait=true --memory=2200 --ha -v=7 --alsologtostderr --kubernetes-version=v1.29.0 --driver=docker --container-runtime=docker --alsologtostderr -v=7: (1m59.953190244s) ha_test.go:107: (dbg) Run: out/minikube -p ha-644161 status -v=7 --alsologtostderr === RUN TestHA/serial/DeployApp ha_test.go:128: (dbg) Run: out/minikube kubectl -p ha-644161 -- apply -f ./testdata/ha/ha-pod-dns-test.yaml ha_test.go:128: (dbg) Done: out/minikube kubectl -p ha-644161 -- apply -f ./testdata/ha/ha-pod-dns-test.yaml: (2.162027834s) ha_test.go:133: (dbg) Run: out/minikube kubectl -p ha-644161 -- rollout status deployment/busybox ha_test.go:133: (dbg) Done: out/minikube kubectl -p ha-644161 -- rollout status deployment/busybox: (4.335988412s) ha_test.go:140: (dbg) Run: out/minikube kubectl -p ha-644161 -- get pods -o jsonpath='{.items[*].status.podIP}' ha_test.go:163: (dbg) Run: out/minikube kubectl -p ha-644161 -- get pods -o jsonpath='{.items[*].metadata.name}' ha_test.go:171: (dbg) Run: out/minikube kubectl -p ha-644161 -- exec busybox-6467d79db7-82czm -- nslookup kubernetes.io ha_test.go:171: (dbg) Run: out/minikube kubectl -p ha-644161 -- exec busybox-6467d79db7-8t78w -- nslookup kubernetes.io ha_test.go:171: (dbg) Run: out/minikube kubectl -p ha-644161 -- exec busybox-6467d79db7-t88hx -- nslookup kubernetes.io ha_test.go:181: (dbg) Run: out/minikube kubectl -p ha-644161 -- exec busybox-6467d79db7-82czm -- nslookup kubernetes.default ha_test.go:181: (dbg) Run: out/minikube kubectl -p ha-644161 -- exec busybox-6467d79db7-8t78w -- nslookup kubernetes.default ha_test.go:181: (dbg) Run: out/minikube kubectl -p ha-644161 -- exec busybox-6467d79db7-t88hx -- nslookup kubernetes.default ha_test.go:189: (dbg) Run: out/minikube kubectl -p ha-644161 -- exec busybox-6467d79db7-82czm -- nslookup kubernetes.default.svc.cluster.local ha_test.go:189: (dbg) Run: out/minikube kubectl -p ha-644161 -- exec busybox-6467d79db7-8t78w -- nslookup kubernetes.default.svc.cluster.local ha_test.go:189: (dbg) Run: out/minikube kubectl -p ha-644161 -- exec busybox-6467d79db7-t88hx -- nslookup kubernetes.default.svc.cluster.local === RUN TestHA/serial/PingHostFromPods ha_test.go:199: (dbg) Run: out/minikube kubectl -p ha-644161 -- get pods -o jsonpath='{.items[*].metadata.name}' ha_test.go:207: (dbg) Run: out/minikube kubectl -p ha-644161 -- exec busybox-6467d79db7-82czm -- sh -c "nslookup host.minikube.internal | awk 'NR==5' | cut -d' ' -f3" ha_test.go:218: (dbg) Run: out/minikube kubectl -p ha-644161 -- exec busybox-6467d79db7-82czm -- sh -c "ping -c 1 192.168.85.1" ha_test.go:207: (dbg) Run: out/minikube kubectl -p ha-644161 -- exec busybox-6467d79db7-8t78w -- sh -c "nslookup host.minikube.internal | awk 'NR==5' | cut -d' ' -f3" ha_test.go:218: (dbg) Run: out/minikube kubectl -p ha-644161 -- exec busybox-6467d79db7-8t78w -- sh -c "ping -c 1 192.168.85.1" ha_test.go:207: (dbg) Run: out/minikube kubectl -p ha-644161 -- exec busybox-6467d79db7-t88hx -- sh -c "nslookup host.minikube.internal | awk 'NR==5' | cut -d' ' -f3" ha_test.go:218: (dbg) Run: out/minikube kubectl -p ha-644161 -- exec busybox-6467d79db7-t88hx -- sh -c "ping -c 1 192.168.85.1" === RUN TestHA/serial/AddWorkerNode ha_test.go:228: (dbg) Run: out/minikube node add -p ha-644161 -v=7 --alsologtostderr ha_test.go:228: (dbg) Done: out/minikube node add -p ha-644161 -v=7 --alsologtostderr: (22.556671629s) ha_test.go:234: (dbg) Run: out/minikube -p ha-644161 status -v=7 --alsologtostderr === RUN TestHA/serial/NodeLabels ha_test.go:255: (dbg) Run: kubectl --context ha-644161 get nodes -o "jsonpath=[{range .items[*]}{.metadata.labels},{end}]" === RUN TestHA/serial/HAppyAfterClusterStart ha_test.go:281: (dbg) Run: out/minikube profile list --output json ha_test.go:281: (dbg) Done: out/minikube profile list --output json: (2.908196406s) === RUN TestHA/serial/CopyFile ha_test.go:326: (dbg) Run: out/minikube -p ha-644161 status --output json -v=7 --alsologtostderr helpers_test.go:556: (dbg) Run: out/minikube -p ha-644161 cp testdata/cp-test.txt ha-644161:/home/docker/cp-test.txt helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161 "sudo cat /home/docker/cp-test.txt" helpers_test.go:556: (dbg) Run: out/minikube -p ha-644161 cp ha-644161:/home/docker/cp-test.txt /tmp/TestHAserialCopyFile1924574508/001/cp-test_ha-644161.txt helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161 "sudo cat /home/docker/cp-test.txt" helpers_test.go:556: (dbg) Run: out/minikube -p ha-644161 cp ha-644161:/home/docker/cp-test.txt ha-644161-m02:/home/docker/cp-test_ha-644161_ha-644161-m02.txt helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161 "sudo cat /home/docker/cp-test.txt" helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m02 "sudo cat /home/docker/cp-test_ha-644161_ha-644161-m02.txt" helpers_test.go:556: (dbg) Run: out/minikube -p ha-644161 cp ha-644161:/home/docker/cp-test.txt ha-644161-m03:/home/docker/cp-test_ha-644161_ha-644161-m03.txt helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161 "sudo cat /home/docker/cp-test.txt" helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m03 "sudo cat /home/docker/cp-test_ha-644161_ha-644161-m03.txt" helpers_test.go:556: (dbg) Run: out/minikube -p ha-644161 cp ha-644161:/home/docker/cp-test.txt ha-644161-m04:/home/docker/cp-test_ha-644161_ha-644161-m04.txt helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161 "sudo cat /home/docker/cp-test.txt" helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m04 "sudo cat /home/docker/cp-test_ha-644161_ha-644161-m04.txt" helpers_test.go:556: (dbg) Run: out/minikube -p ha-644161 cp testdata/cp-test.txt ha-644161-m02:/home/docker/cp-test.txt helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m02 "sudo cat /home/docker/cp-test.txt" helpers_test.go:556: (dbg) Run: out/minikube -p ha-644161 cp ha-644161-m02:/home/docker/cp-test.txt /tmp/TestHAserialCopyFile1924574508/001/cp-test_ha-644161-m02.txt helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m02 "sudo cat /home/docker/cp-test.txt" helpers_test.go:556: (dbg) Run: out/minikube -p ha-644161 cp ha-644161-m02:/home/docker/cp-test.txt ha-644161:/home/docker/cp-test_ha-644161-m02_ha-644161.txt helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m02 "sudo cat /home/docker/cp-test.txt" helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161 "sudo cat /home/docker/cp-test_ha-644161-m02_ha-644161.txt" helpers_test.go:556: (dbg) Run: out/minikube -p ha-644161 cp ha-644161-m02:/home/docker/cp-test.txt ha-644161-m03:/home/docker/cp-test_ha-644161-m02_ha-644161-m03.txt helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m02 "sudo cat /home/docker/cp-test.txt" helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m03 "sudo cat /home/docker/cp-test_ha-644161-m02_ha-644161-m03.txt" helpers_test.go:556: (dbg) Run: out/minikube -p ha-644161 cp ha-644161-m02:/home/docker/cp-test.txt ha-644161-m04:/home/docker/cp-test_ha-644161-m02_ha-644161-m04.txt helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m02 "sudo cat /home/docker/cp-test.txt" helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m04 "sudo cat /home/docker/cp-test_ha-644161-m02_ha-644161-m04.txt" helpers_test.go:556: (dbg) Run: out/minikube -p ha-644161 cp testdata/cp-test.txt ha-644161-m03:/home/docker/cp-test.txt helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m03 "sudo cat /home/docker/cp-test.txt" helpers_test.go:556: (dbg) Run: out/minikube -p ha-644161 cp ha-644161-m03:/home/docker/cp-test.txt /tmp/TestHAserialCopyFile1924574508/001/cp-test_ha-644161-m03.txt helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m03 "sudo cat /home/docker/cp-test.txt" helpers_test.go:556: (dbg) Run: out/minikube -p ha-644161 cp ha-644161-m03:/home/docker/cp-test.txt ha-644161:/home/docker/cp-test_ha-644161-m03_ha-644161.txt helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m03 "sudo cat /home/docker/cp-test.txt" helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161 "sudo cat /home/docker/cp-test_ha-644161-m03_ha-644161.txt" helpers_test.go:556: (dbg) Run: out/minikube -p ha-644161 cp ha-644161-m03:/home/docker/cp-test.txt ha-644161-m02:/home/docker/cp-test_ha-644161-m03_ha-644161-m02.txt helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m03 "sudo cat /home/docker/cp-test.txt" helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m02 "sudo cat /home/docker/cp-test_ha-644161-m03_ha-644161-m02.txt" helpers_test.go:556: (dbg) Run: out/minikube -p ha-644161 cp ha-644161-m03:/home/docker/cp-test.txt ha-644161-m04:/home/docker/cp-test_ha-644161-m03_ha-644161-m04.txt helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m03 "sudo cat /home/docker/cp-test.txt" helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m04 "sudo cat /home/docker/cp-test_ha-644161-m03_ha-644161-m04.txt" helpers_test.go:556: (dbg) Run: out/minikube -p ha-644161 cp testdata/cp-test.txt ha-644161-m04:/home/docker/cp-test.txt helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m04 "sudo cat /home/docker/cp-test.txt" helpers_test.go:556: (dbg) Run: out/minikube -p ha-644161 cp ha-644161-m04:/home/docker/cp-test.txt /tmp/TestHAserialCopyFile1924574508/001/cp-test_ha-644161-m04.txt helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m04 "sudo cat /home/docker/cp-test.txt" helpers_test.go:556: (dbg) Run: out/minikube -p ha-644161 cp ha-644161-m04:/home/docker/cp-test.txt ha-644161:/home/docker/cp-test_ha-644161-m04_ha-644161.txt helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m04 "sudo cat /home/docker/cp-test.txt" helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161 "sudo cat /home/docker/cp-test_ha-644161-m04_ha-644161.txt" helpers_test.go:556: (dbg) Run: out/minikube -p ha-644161 cp ha-644161-m04:/home/docker/cp-test.txt ha-644161-m02:/home/docker/cp-test_ha-644161-m04_ha-644161-m02.txt helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m04 "sudo cat /home/docker/cp-test.txt" helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m02 "sudo cat /home/docker/cp-test_ha-644161-m04_ha-644161-m02.txt" helpers_test.go:556: (dbg) Run: out/minikube -p ha-644161 cp ha-644161-m04:/home/docker/cp-test.txt ha-644161-m03:/home/docker/cp-test_ha-644161-m04_ha-644161-m03.txt helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m04 "sudo cat /home/docker/cp-test.txt" helpers_test.go:534: (dbg) Run: out/minikube -p ha-644161 ssh -n ha-644161-m03 "sudo cat /home/docker/cp-test_ha-644161-m04_ha-644161-m03.txt" === RUN TestHA/serial/StopSecondaryNode ha_test.go:363: (dbg) Run: out/minikube -p ha-644161 node stop m02 -v=7 --alsologtostderr ha_test.go:363: (dbg) Done: out/minikube -p ha-644161 node stop m02 -v=7 --alsologtostderr: (11.541144838s) ha_test.go:369: (dbg) Run: out/minikube -p ha-644161 status -v=7 --alsologtostderr ha_test.go:369: (dbg) Non-zero exit: out/minikube -p ha-644161 status -v=7 --alsologtostderr: exit status 7 (537.525477ms) -- stdout -- ha-644161 type: Control Plane host: Running kubelet: Running apiserver: Running kubeconfig: Configured ha-644161-m02 type: Control Plane host: Stopped kubelet: Stopped apiserver: Stopped kubeconfig: Stopped ha-644161-m03 type: Control Plane host: Running kubelet: Running apiserver: Running kubeconfig: Configured ha-644161-m04 type: Worker host: Running kubelet: Running -- /stdout -- ** stderr ** I0109 23:31:06.882122 944811 out.go:296] Setting OutFile to fd 1 ... I0109 23:31:06.882221 944811 out.go:348] isatty.IsTerminal(1) = false I0109 23:31:06.882226 944811 out.go:309] Setting ErrFile to fd 2... I0109 23:31:06.882234 944811 out.go:348] isatty.IsTerminal(2) = false I0109 23:31:06.882398 944811 root.go:338] Updating PATH: /home/prezha/.minikube/bin I0109 23:31:06.882528 944811 out.go:303] Setting JSON to false I0109 23:31:06.882542 944811 mustload.go:65] Loading cluster: ha-644161 I0109 23:31:06.882661 944811 notify.go:220] Checking for updates... I0109 23:31:06.882870 944811 config.go:182] Loaded profile config "ha-644161": Driver=docker, ContainerRuntime=docker, KubernetesVersion=v1.29.0 I0109 23:31:06.882879 944811 status.go:255] checking status of ha-644161 ... I0109 23:31:06.883240 944811 cli_runner.go:164] Run: docker container inspect ha-644161 --format={{.State.Status}} I0109 23:31:06.895813 944811 status.go:330] ha-644161 host status = "Running" (err=) I0109 23:31:06.895824 944811 host.go:66] Checking if "ha-644161" exists ... I0109 23:31:06.896047 944811 cli_runner.go:164] Run: docker container inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}" ha-644161 I0109 23:31:06.908688 944811 host.go:66] Checking if "ha-644161" exists ... I0109 23:31:06.908894 944811 ssh_runner.go:195] Run: sh -c "df -h /var | awk 'NR==2{print $5}'" I0109 23:31:06.908927 944811 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-644161 I0109 23:31:06.923835 944811 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:32887 SSHKeyPath:/home/prezha/.minikube/machines/ha-644161/id_rsa Username:docker} I0109 23:31:07.024201 944811 ssh_runner.go:195] Run: systemctl --version I0109 23:31:07.027398 944811 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service kubelet I0109 23:31:07.037709 944811 kubeconfig.go:125] found "ha-644161" server: "https://192.168.85.254:8443" I0109 23:31:07.037729 944811 api_server.go:166] Checking apiserver status ... I0109 23:31:07.037762 944811 ssh_runner.go:195] Run: sudo pgrep -xnf kube-apiserver.*minikube.* I0109 23:31:07.048357 944811 ssh_runner.go:195] Run: sudo egrep ^[0-9]+:freezer: /proc/2420/cgroup W0109 23:31:07.057382 944811 api_server.go:177] unable to find freezer cgroup: sudo egrep ^[0-9]+:freezer: /proc/2420/cgroup: Process exited with status 1 stdout: stderr: I0109 23:31:07.057417 944811 ssh_runner.go:195] Run: ls I0109 23:31:07.060246 944811 api_server.go:253] Checking apiserver healthz at https://192.168.85.254:8443/healthz ... I0109 23:31:07.063840 944811 api_server.go:279] https://192.168.85.254:8443/healthz returned 200: ok I0109 23:31:07.063857 944811 status.go:424] ha-644161 apiserver status = Running (err=) I0109 23:31:07.063866 944811 status.go:257] ha-644161 status: &{Name:ha-644161 Host:Running Kubelet:Running APIServer:Running Kubeconfig:Configured Worker:false TimeToStop: DockerEnv: PodManEnv:} I0109 23:31:07.063883 944811 status.go:255] checking status of ha-644161-m02 ... I0109 23:31:07.064090 944811 cli_runner.go:164] Run: docker container inspect ha-644161-m02 --format={{.State.Status}} I0109 23:31:07.077093 944811 status.go:330] ha-644161-m02 host status = "Stopped" (err=) I0109 23:31:07.077106 944811 status.go:343] host is not running, skipping remaining checks I0109 23:31:07.077111 944811 status.go:257] ha-644161-m02 status: &{Name:ha-644161-m02 Host:Stopped Kubelet:Stopped APIServer:Stopped Kubeconfig:Stopped Worker:false TimeToStop: DockerEnv: PodManEnv:} I0109 23:31:07.077123 944811 status.go:255] checking status of ha-644161-m03 ... I0109 23:31:07.077315 944811 cli_runner.go:164] Run: docker container inspect ha-644161-m03 --format={{.State.Status}} I0109 23:31:07.089980 944811 status.go:330] ha-644161-m03 host status = "Running" (err=) I0109 23:31:07.090000 944811 host.go:66] Checking if "ha-644161-m03" exists ... I0109 23:31:07.090304 944811 cli_runner.go:164] Run: docker container inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}" ha-644161-m03 I0109 23:31:07.103696 944811 host.go:66] Checking if "ha-644161-m03" exists ... I0109 23:31:07.103960 944811 ssh_runner.go:195] Run: sh -c "df -h /var | awk 'NR==2{print $5}'" I0109 23:31:07.103996 944811 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-644161-m03 I0109 23:31:07.116475 944811 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:32897 SSHKeyPath:/home/prezha/.minikube/machines/ha-644161-m03/id_rsa Username:docker} I0109 23:31:07.210246 944811 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service kubelet I0109 23:31:07.220371 944811 kubeconfig.go:125] found "ha-644161" server: "https://192.168.85.254:8443" I0109 23:31:07.220387 944811 api_server.go:166] Checking apiserver status ... I0109 23:31:07.220413 944811 ssh_runner.go:195] Run: sudo pgrep -xnf kube-apiserver.*minikube.* I0109 23:31:07.230529 944811 ssh_runner.go:195] Run: sudo egrep ^[0-9]+:freezer: /proc/2284/cgroup W0109 23:31:07.238339 944811 api_server.go:177] unable to find freezer cgroup: sudo egrep ^[0-9]+:freezer: /proc/2284/cgroup: Process exited with status 1 stdout: stderr: I0109 23:31:07.238374 944811 ssh_runner.go:195] Run: ls I0109 23:31:07.241301 944811 api_server.go:253] Checking apiserver healthz at https://192.168.85.254:8443/healthz ... I0109 23:31:07.244736 944811 api_server.go:279] https://192.168.85.254:8443/healthz returned 200: ok I0109 23:31:07.244751 944811 status.go:424] ha-644161-m03 apiserver status = Running (err=) I0109 23:31:07.244757 944811 status.go:257] ha-644161-m03 status: &{Name:ha-644161-m03 Host:Running Kubelet:Running APIServer:Running Kubeconfig:Configured Worker:false TimeToStop: DockerEnv: PodManEnv:} I0109 23:31:07.244772 944811 status.go:255] checking status of ha-644161-m04 ... I0109 23:31:07.244966 944811 cli_runner.go:164] Run: docker container inspect ha-644161-m04 --format={{.State.Status}} I0109 23:31:07.257449 944811 status.go:330] ha-644161-m04 host status = "Running" (err=) I0109 23:31:07.257463 944811 host.go:66] Checking if "ha-644161-m04" exists ... I0109 23:31:07.257718 944811 cli_runner.go:164] Run: docker container inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}" ha-644161-m04 I0109 23:31:07.269872 944811 host.go:66] Checking if "ha-644161-m04" exists ... I0109 23:31:07.270117 944811 ssh_runner.go:195] Run: sh -c "df -h /var | awk 'NR==2{print $5}'" I0109 23:31:07.270229 944811 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-644161-m04 I0109 23:31:07.282895 944811 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:32902 SSHKeyPath:/home/prezha/.minikube/machines/ha-644161-m04/id_rsa Username:docker} I0109 23:31:07.373958 944811 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service kubelet I0109 23:31:07.383974 944811 status.go:257] ha-644161-m04 status: &{Name:ha-644161-m04 Host:Running Kubelet:Running APIServer:Irrelevant Kubeconfig:Irrelevant Worker:true TimeToStop: DockerEnv: PodManEnv:} ** /stderr ** === RUN TestHA/serial/DegradedAfterControlPlaneNodeStop ha_test.go:390: (dbg) Run: out/minikube profile list --output json ha_test.go:390: (dbg) Done: out/minikube profile list --output json: (1.956797634s) === RUN TestHA/serial/RestartSecondaryNode ha_test.go:420: (dbg) Run: out/minikube -p ha-644161 node start m02 -v=7 --alsologtostderr ha_test.go:420: (dbg) Done: out/minikube -p ha-644161 node start m02 -v=7 --alsologtostderr: (19.490372075s) ha_test.go:428: (dbg) Run: out/minikube -p ha-644161 status -v=7 --alsologtostderr ha_test.go:448: (dbg) Run: kubectl get nodes === RUN TestHA/serial/HAppyAfterSecondaryNodeRestart ha_test.go:281: (dbg) Run: out/minikube profile list --output json ha_test.go:281: (dbg) Done: out/minikube profile list --output json: (9.508901901s) === RUN TestHA/serial/RestartClusterKeepsNodes ha_test.go:456: (dbg) Run: out/minikube node list -p ha-644161 -v=7 --alsologtostderr ha_test.go:462: (dbg) Run: out/minikube stop -p ha-644161 -v=7 --alsologtostderr ha_test.go:462: (dbg) Done: out/minikube stop -p ha-644161 -v=7 --alsologtostderr: (36.650643553s) ha_test.go:467: (dbg) Run: out/minikube start -p ha-644161 --wait=true -v=7 --alsologtostderr ha_test.go:467: (dbg) Done: out/minikube start -p ha-644161 --wait=true -v=7 --alsologtostderr: (1m53.889404684s) ha_test.go:472: (dbg) Run: out/minikube node list -p ha-644161 === RUN TestHA/serial/DeleteSecondaryNode ha_test.go:487: (dbg) Run: out/minikube -p ha-644161 node delete m03 -v=7 --alsologtostderr ha_test.go:487: (dbg) Done: out/minikube -p ha-644161 node delete m03 -v=7 --alsologtostderr: (10.690521659s) ha_test.go:493: (dbg) Run: out/minikube -p ha-644161 status -v=7 --alsologtostderr ha_test.go:511: (dbg) Run: kubectl get nodes ha_test.go:519: (dbg) Run: kubectl get nodes -o "go-template='{{range .items}}{{range .status.conditions}}{{if eq .type "Ready"}} {{.status}}{{"\n"}}{{end}}{{end}}{{end}}'" === RUN TestHA/serial/DegradedAfterSecondaryNodeDelete ha_test.go:390: (dbg) Run: out/minikube profile list --output json ha_test.go:390: (dbg) Done: out/minikube profile list --output json: (1.895477061s) === RUN TestHA/serial/StopCluster ha_test.go:531: (dbg) Run: out/minikube -p ha-644161 stop -v=7 --alsologtostderr ha_test.go:531: (dbg) Done: out/minikube -p ha-644161 stop -v=7 --alsologtostderr: (34.95372136s) ha_test.go:537: (dbg) Run: out/minikube -p ha-644161 status -v=7 --alsologtostderr ha_test.go:537: (dbg) Non-zero exit: out/minikube -p ha-644161 status -v=7 --alsologtostderr: exit status 7 (78.608001ms) -- stdout -- ha-644161 type: Control Plane host: Stopped kubelet: Stopped apiserver: Stopped kubeconfig: Stopped ha-644161-m02 type: Control Plane host: Stopped kubelet: Stopped apiserver: Stopped kubeconfig: Stopped ha-644161-m04 type: Worker host: Stopped kubelet: Stopped -- /stdout -- ** stderr ** I0109 23:34:57.853170 1043209 out.go:296] Setting OutFile to fd 1 ... I0109 23:34:57.853259 1043209 out.go:348] isatty.IsTerminal(1) = false I0109 23:34:57.853264 1043209 out.go:309] Setting ErrFile to fd 2... I0109 23:34:57.853269 1043209 out.go:348] isatty.IsTerminal(2) = false I0109 23:34:57.853415 1043209 root.go:338] Updating PATH: /home/prezha/.minikube/bin I0109 23:34:57.853613 1043209 out.go:303] Setting JSON to false I0109 23:34:57.853628 1043209 mustload.go:65] Loading cluster: ha-644161 I0109 23:34:57.853724 1043209 notify.go:220] Checking for updates... I0109 23:34:57.853941 1043209 config.go:182] Loaded profile config "ha-644161": Driver=docker, ContainerRuntime=docker, KubernetesVersion=v1.29.0 I0109 23:34:57.853949 1043209 status.go:255] checking status of ha-644161 ... I0109 23:34:57.854273 1043209 cli_runner.go:164] Run: docker container inspect ha-644161 --format={{.State.Status}} I0109 23:34:57.868057 1043209 status.go:330] ha-644161 host status = "Stopped" (err=) I0109 23:34:57.868090 1043209 status.go:343] host is not running, skipping remaining checks I0109 23:34:57.868096 1043209 status.go:257] ha-644161 status: &{Name:ha-644161 Host:Stopped Kubelet:Stopped APIServer:Stopped Kubeconfig:Stopped Worker:false TimeToStop: DockerEnv: PodManEnv:} I0109 23:34:57.868119 1043209 status.go:255] checking status of ha-644161-m02 ... I0109 23:34:57.868368 1043209 cli_runner.go:164] Run: docker container inspect ha-644161-m02 --format={{.State.Status}} I0109 23:34:57.882271 1043209 status.go:330] ha-644161-m02 host status = "Stopped" (err=) I0109 23:34:57.882288 1043209 status.go:343] host is not running, skipping remaining checks I0109 23:34:57.882296 1043209 status.go:257] ha-644161-m02 status: &{Name:ha-644161-m02 Host:Stopped Kubelet:Stopped APIServer:Stopped Kubeconfig:Stopped Worker:false TimeToStop: DockerEnv: PodManEnv:} I0109 23:34:57.882315 1043209 status.go:255] checking status of ha-644161-m04 ... I0109 23:34:57.882616 1043209 cli_runner.go:164] Run: docker container inspect ha-644161-m04 --format={{.State.Status}} I0109 23:34:57.895683 1043209 status.go:330] ha-644161-m04 host status = "Stopped" (err=) I0109 23:34:57.895696 1043209 status.go:343] host is not running, skipping remaining checks I0109 23:34:57.895702 1043209 status.go:257] ha-644161-m04 status: &{Name:ha-644161-m04 Host:Stopped Kubelet:Stopped APIServer:Stopped Kubeconfig:Stopped Worker:true TimeToStop: DockerEnv: PodManEnv:} ** /stderr ** === RUN TestHA/serial/RestartCluster ha_test.go:560: (dbg) Run: out/minikube start -p ha-644161 --wait=true -v=7 --alsologtostderr --kubernetes-version=v1.29.0 --driver=docker --container-runtime=docker --alsologtostderr -v=7 ha_test.go:560: (dbg) Done: out/minikube start -p ha-644161 --wait=true -v=7 --alsologtostderr --kubernetes-version=v1.29.0 --driver=docker --container-runtime=docker --alsologtostderr -v=7: (1m23.22141728s) ha_test.go:566: (dbg) Run: out/minikube -p ha-644161 status -v=7 --alsologtostderr ha_test.go:584: (dbg) Run: kubectl get nodes ha_test.go:592: (dbg) Run: kubectl get nodes -o "go-template='{{range .items}}{{range .status.conditions}}{{if eq .type "Ready"}} {{.status}}{{"\n"}}{{end}}{{end}}{{end}}'" === RUN TestHA/serial/DegradedAfterClusterRestart ha_test.go:390: (dbg) Run: out/minikube profile list --output json ha_test.go:390: (dbg) Done: out/minikube profile list --output json: (1.991364829s) === RUN TestHA/serial/AddSecondaryNode ha_test.go:605: (dbg) Run: out/minikube node add -p ha-644161 --control-plane -v=7 --alsologtostderr ha_test.go:605: (dbg) Done: out/minikube node add -p ha-644161 --control-plane -v=7 --alsologtostderr: (39.111082775s) ha_test.go:611: (dbg) Run: out/minikube -p ha-644161 status -v=7 --alsologtostderr === RUN TestHA/serial/HAppyAfterSecondaryNodeAdd ha_test.go:281: (dbg) Run: out/minikube profile list --output json ha_test.go:281: (dbg) Done: out/minikube profile list --output json: (2.232054282s) === NAME TestHA helpers_test.go:183: skipping cleanup of ha-644161 (--cleanup=false) --- PASS: TestHA (541.57s) --- PASS: TestHA/serial (541.56s) --- PASS: TestHA/serial/StartCluster (120.50s) --- PASS: TestHA/serial/DeployApp (8.36s) --- PASS: TestHA/serial/PingHostFromPods (1.12s) --- PASS: TestHA/serial/AddWorkerNode (23.31s) --- PASS: TestHA/serial/NodeLabels (0.06s) --- PASS: TestHA/serial/HAppyAfterClusterStart (2.91s) --- PASS: TestHA/serial/CopyFile (14.71s) --- PASS: TestHA/serial/StopSecondaryNode (12.08s) --- PASS: TestHA/serial/DegradedAfterControlPlaneNodeStop (1.96s) --- PASS: TestHA/serial/RestartSecondaryNode (20.22s) --- PASS: TestHA/serial/HAppyAfterSecondaryNodeRestart (9.51s) --- PASS: TestHA/serial/RestartClusterKeepsNodes (150.62s) --- PASS: TestHA/serial/DeleteSecondaryNode (11.28s) --- PASS: TestHA/serial/DegradedAfterSecondaryNodeDelete (1.90s) --- PASS: TestHA/serial/StopCluster (35.03s) --- PASS: TestHA/serial/RestartCluster (83.91s) --- PASS: TestHA/serial/DegradedAfterClusterRestart (1.99s) --- PASS: TestHA/serial/AddSecondaryNode (39.86s) --- PASS: TestHA/serial/HAppyAfterSecondaryNodeAdd (2.23s) PASS Tests completed in 9m1.572406403s (result code 0) ok k8s.io/minikube/test/integration 541.607s [prezha@codex minikube (ha)]$