From 677486928cc007814f5eb846eb876c0e7b32f5bc Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Thu, 11 Oct 2018 12:36:08 -0700 Subject: [PATCH 1/3] asset/ignition: move content under bootstrap The content package only contains bootstrap assets. It's more clear that that is the case if that package is under the bootstrap package, rather than being adjacent to it. --- pkg/asset/ignition/bootstrap/bootstrap.go | 2 +- pkg/asset/ignition/{ => bootstrap}/content/bootkube.go | 0 pkg/asset/ignition/{ => bootstrap}/content/doc.go | 2 +- pkg/asset/ignition/{ => bootstrap}/content/kubelet.go | 0 pkg/asset/ignition/{ => bootstrap}/content/tectonic.go | 0 5 files changed, 2 insertions(+), 2 deletions(-) rename pkg/asset/ignition/{ => bootstrap}/content/bootkube.go (100%) rename pkg/asset/ignition/{ => bootstrap}/content/doc.go (73%) rename pkg/asset/ignition/{ => bootstrap}/content/kubelet.go (100%) rename pkg/asset/ignition/{ => bootstrap}/content/tectonic.go (100%) diff --git a/pkg/asset/ignition/bootstrap/bootstrap.go b/pkg/asset/ignition/bootstrap/bootstrap.go index 8be09a3388a..b69077dd689 100644 --- a/pkg/asset/ignition/bootstrap/bootstrap.go +++ b/pkg/asset/ignition/bootstrap/bootstrap.go @@ -15,7 +15,7 @@ import ( "github.com/openshift/installer/pkg/asset" "github.com/openshift/installer/pkg/asset/ignition" - "github.com/openshift/installer/pkg/asset/ignition/content" + "github.com/openshift/installer/pkg/asset/ignition/bootstrap/content" "github.com/openshift/installer/pkg/asset/installconfig" "github.com/openshift/installer/pkg/asset/kubeconfig" "github.com/openshift/installer/pkg/asset/manifests" diff --git a/pkg/asset/ignition/content/bootkube.go b/pkg/asset/ignition/bootstrap/content/bootkube.go similarity index 100% rename from pkg/asset/ignition/content/bootkube.go rename to pkg/asset/ignition/bootstrap/content/bootkube.go diff --git a/pkg/asset/ignition/content/doc.go b/pkg/asset/ignition/bootstrap/content/doc.go similarity index 73% rename from pkg/asset/ignition/content/doc.go rename to pkg/asset/ignition/bootstrap/content/doc.go index caaafb25a8f..9dceac33402 100644 --- a/pkg/asset/ignition/content/doc.go +++ b/pkg/asset/ignition/bootstrap/content/doc.go @@ -1,3 +1,3 @@ // Package content contains the contents of files and systemd units to be added -// to Ignition configs. +// to bootstrap Ignition configs. package content diff --git a/pkg/asset/ignition/content/kubelet.go b/pkg/asset/ignition/bootstrap/content/kubelet.go similarity index 100% rename from pkg/asset/ignition/content/kubelet.go rename to pkg/asset/ignition/bootstrap/content/kubelet.go diff --git a/pkg/asset/ignition/content/tectonic.go b/pkg/asset/ignition/bootstrap/content/tectonic.go similarity index 100% rename from pkg/asset/ignition/content/tectonic.go rename to pkg/asset/ignition/bootstrap/content/tectonic.go From f0db7a7592a18535f099fab39867258edad7146f Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Thu, 11 Oct 2018 12:52:26 -0700 Subject: [PATCH 2/3] asset/ignition: clean up bootstrap content Adding a line break before and after the closing and opening grave ticks makes this a little more readable and is the style that the rest of the codebase uses. This does result in an empty line at the top and bottom of the file, which is why this style cannot be applied to the shell scripts. --- .../ignition/bootstrap/content/bootkube.go | 41 ++++++++++--------- .../ignition/bootstrap/content/kubelet.go | 6 ++- .../ignition/bootstrap/content/tectonic.go | 9 ++-- 3 files changed, 32 insertions(+), 24 deletions(-) diff --git a/pkg/asset/ignition/bootstrap/content/bootkube.go b/pkg/asset/ignition/bootstrap/content/bootkube.go index 8dc4b56a25a..60c014ad0cc 100644 --- a/pkg/asset/ignition/bootstrap/content/bootkube.go +++ b/pkg/asset/ignition/bootstrap/content/bootkube.go @@ -7,7 +7,8 @@ import ( const ( // BootkubeSystemdContents is a service for running bootkube on the bootstrap // nodes - BootkubeSystemdContents = `[Unit] + BootkubeSystemdContents = ` +[Unit] Description=Bootstrap a Kubernetes cluster Wants=kubelet.service After=kubelet.service @@ -18,7 +19,8 @@ WorkingDirectory=/opt/tectonic ExecStart=/opt/tectonic/bootkube.sh Restart=on-failure -RestartSec=5s` +RestartSec=5s +` ) var ( @@ -113,22 +115,22 @@ fi if [ ! -d kube-scheduler-bootstrap ] then - echo "Rendering Kubernetes Scheduler core manifests..." - - # shellcheck disable=SC2154 - podman run \ - --volume "$PWD:/assets:z" \ - "${KUBE_SCHEDULER_OPERATOR_IMAGE}" \ - /usr/bin/cluster-kube-scheduler-operator render \ - --asset-input-dir=/assets/tls \ - --asset-output-dir=/assets/kube-scheduler-bootstrap \ - --config-override-file=/usr/share/bootkube/manifests/config/config-overrides.yaml \ - --config-output-file=/assets/kube-scheduler-bootstrap/config - - # TODO: copy the bootstrap manifests to replace kube-core-operator - cp --recursive kube-scheduler-bootstrap/manifests/00_openshift-kube-scheduler-ns.yaml manifests/00_openshift-kube-scheduler-ns.yaml - cp --recursive kube-scheduler-bootstrap/manifests/secret-* manifests/ - cp --recursive kube-scheduler-bootstrap/manifests/configmap-* manifests/ + echo "Rendering Kubernetes Scheduler core manifests..." + + # shellcheck disable=SC2154 + podman run \ + --volume "$PWD:/assets:z" \ + "${KUBE_SCHEDULER_OPERATOR_IMAGE}" \ + /usr/bin/cluster-kube-scheduler-operator render \ + --asset-input-dir=/assets/tls \ + --asset-output-dir=/assets/kube-scheduler-bootstrap \ + --config-override-file=/usr/share/bootkube/manifests/config/config-overrides.yaml \ + --config-output-file=/assets/kube-scheduler-bootstrap/config + + # TODO: copy the bootstrap manifests to replace kube-core-operator + cp --recursive kube-scheduler-bootstrap/manifests/00_openshift-kube-scheduler-ns.yaml manifests/00_openshift-kube-scheduler-ns.yaml + cp --recursive kube-scheduler-bootstrap/manifests/secret-* manifests/ + cp --recursive kube-scheduler-bootstrap/manifests/configmap-* manifests/ fi if [ ! -d mco-bootstrap ] @@ -230,5 +232,6 @@ podman run \ --network=host \ --entrypoint=/bootkube \ "{{.BootkubeImage}}" \ - start --asset-dir=/assets`)) + start --asset-dir=/assets +`)) ) diff --git a/pkg/asset/ignition/bootstrap/content/kubelet.go b/pkg/asset/ignition/bootstrap/content/kubelet.go index 52e881f2dff..3c533cb0bd6 100644 --- a/pkg/asset/ignition/bootstrap/content/kubelet.go +++ b/pkg/asset/ignition/bootstrap/content/kubelet.go @@ -7,7 +7,8 @@ import ( var ( // KubeletSystemdTemplate is a service for running the kubelet on the // bootstrap nodes. - KubeletSystemdTemplate = template.Must(template.New("kubelet.service").Parse(`[Unit] + KubeletSystemdTemplate = template.Must(template.New("kubelet.service").Parse(` +[Unit] Description=Kubernetes Kubelet Wants=rpc-statd.service @@ -46,5 +47,6 @@ Restart=always RestartSec=10 [Install] -WantedBy=multi-user.target`)) +WantedBy=multi-user.target +`)) ) diff --git a/pkg/asset/ignition/bootstrap/content/tectonic.go b/pkg/asset/ignition/bootstrap/content/tectonic.go index a37da6937a3..6562fd42ba1 100644 --- a/pkg/asset/ignition/bootstrap/content/tectonic.go +++ b/pkg/asset/ignition/bootstrap/content/tectonic.go @@ -2,7 +2,8 @@ package content const ( // TectonicSystemdContents is a service that runs tectonic on the masters. - TectonicSystemdContents = `[Unit] + TectonicSystemdContents = ` +[Unit] Description=Bootstrap a Tectonic cluster Wants=bootkube.service After=bootkube.service @@ -16,7 +17,8 @@ Restart=on-failure RestartSec=5s [Install] -WantedBy=multi-user.target` +WantedBy=multi-user.target +` // TectonicShFileContents is a script file for running tectonic on bootstrap // nodes. @@ -90,5 +92,6 @@ done # Wait for Tectonic pods wait_for_pods tectonic-system -echo "Tectonic installation is done"` +echo "Tectonic installation is done" +` ) From 3dbbbf69d5f6eb96ce53f16e93977475b40158aa Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Fri, 12 Oct 2018 12:30:33 -0700 Subject: [PATCH 3/3] asset/ignition: add event on bootstrap completion In order to allow the installer to eventually track the installation progress, this adds a reporting service which inserts an event when the bootstrap process has completed. This event can be used to determine when it is safe to remove the bootstrap node, since it signifies that the bootstrap control plane has pivoted to the scheduled control plane and that all of the OpenShift manifests have been inserted. This no longer enables `tectonic.service` because `progress.service` is enabled and lists both `tectonic.service` and `bootkube.service` as dependencies. systemd will start all three services as a result. In an ideal world, this would make use of sd-notify to signal the completion of prerequisites but this doesn't work for two reasons. First, runc doesn't support passing the extra file descriptor used by sd-notify [1]. Second, systemd doesn't support retrying activations when a dependency has failed [2]. This change makes use of "breadcrumb" files to signal completion as a workaround. In the creation of the event, `date` is invoked and passed a custom format. This was necessary because Golang requires the `T` separator in the time package [3] but GNU's `date` uses a space separator. ISO8601 was also tried, but Golang doesn't seem to parse that correctly either (`date` uses "+0000" to specify timezone but Golang expects "+00:00"). [1]: https://github.com/opencontainers/runc/pull/1807 [2]: https://github.com/systemd/systemd/issues/1312 [3]: https://github.com/golang/go/issues/25937 --- pkg/asset/ignition/bootstrap/bootstrap.go | 7 ++- .../ignition/bootstrap/content/bootkube.go | 2 + .../ignition/bootstrap/content/report.go | 54 +++++++++++++++++++ .../ignition/bootstrap/content/tectonic.go | 5 +- 4 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 pkg/asset/ignition/bootstrap/content/report.go diff --git a/pkg/asset/ignition/bootstrap/bootstrap.go b/pkg/asset/ignition/bootstrap/bootstrap.go index b69077dd689..61cc881be1c 100644 --- a/pkg/asset/ignition/bootstrap/bootstrap.go +++ b/pkg/asset/ignition/bootstrap/bootstrap.go @@ -103,7 +103,8 @@ func (a *Bootstrap) Generate(dependencies asset.Parents) error { a.Config.Systemd.Units = append( a.Config.Systemd.Units, igntypes.Unit{Name: "bootkube.service", Contents: content.BootkubeSystemdContents}, - igntypes.Unit{Name: "tectonic.service", Contents: content.TectonicSystemdContents, Enabled: util.BoolToPtr(true)}, + igntypes.Unit{Name: "tectonic.service", Contents: content.TectonicSystemdContents}, + igntypes.Unit{Name: "progress.service", Contents: content.ReportSystemdContents, Enabled: util.BoolToPtr(true)}, igntypes.Unit{Name: "kubelet.service", Contents: applyTemplateData(content.KubeletSystemdTemplate, templateData), Enabled: util.BoolToPtr(true)}, ) @@ -183,6 +184,10 @@ func (a *Bootstrap) addBootstrapFiles(dependencies asset.Parents) { a.Config.Storage.Files, ignition.FilesFromAsset(rootDir, 0644, kubeCoreOperator)..., ) + a.Config.Storage.Files = append( + a.Config.Storage.Files, + ignition.FileFromString("/opt/tectonic/report-progress.sh", 0555, content.ReportShFileContents), + ) } func (a *Bootstrap) addBootkubeFiles(dependencies asset.Parents, templateData *bootstrapTemplateData) { diff --git a/pkg/asset/ignition/bootstrap/content/bootkube.go b/pkg/asset/ignition/bootstrap/content/bootkube.go index 60c014ad0cc..0180cd99391 100644 --- a/pkg/asset/ignition/bootstrap/content/bootkube.go +++ b/pkg/asset/ignition/bootstrap/content/bootkube.go @@ -17,6 +17,8 @@ After=kubelet.service WorkingDirectory=/opt/tectonic ExecStart=/opt/tectonic/bootkube.sh +# Workaround for https://github.com/opencontainers/runc/pull/1807 +ExecStartPost=/usr/bin/touch /opt/tectonic/.bootkube.done Restart=on-failure RestartSec=5s diff --git a/pkg/asset/ignition/bootstrap/content/report.go b/pkg/asset/ignition/bootstrap/content/report.go new file mode 100644 index 00000000000..5178c2be5ab --- /dev/null +++ b/pkg/asset/ignition/bootstrap/content/report.go @@ -0,0 +1,54 @@ +package content + +const ( + // ReportSystemdContents is a service that reports the bootstrap progress + // via a Kubernetes Event. + ReportSystemdContents = ` +[Unit] +Description=Report the completion of the cluster bootstrap process +# Workaround for https://github.com/systemd/systemd/issues/1312 +Wants=bootkube.service tectonic.service +After=bootkube.service tectonic.service + +[Service] +# Workaround for https://github.com/systemd/systemd/issues/1312 and https://github.com/opencontainers/runc/pull/1807 +ExecStartPre=/usr/bin/test -f /opt/tectonic/.bootkube.done +ExecStartPre=/usr/bin/test -f /opt/tectonic/.tectonic.done +ExecStart=/opt/tectonic/report-progress.sh /opt/tectonic/auth/kubeconfig bootstrap-complete "cluster bootstrapping has completed" + +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=multi-user.target +` + + // ReportShFileContents is a script for reporting the bootstrap progress. + ReportShFileContents = `#!/usr/bin/env bash +set -e + +KUBECONFIG="${1}" +NAME="${2}" +MESSAGE="${3}" +TIMESTAMP="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" + +echo "Reporting install progress..." + +oc --config="$KUBECONFIG" create <