From d139a665575d42af90af50f95ca3e48563fdcff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20L=C3=BCke?= Date: Mon, 19 Apr 2021 12:43:07 +0200 Subject: [PATCH 1/4] platform: allow CLC templating for dynamic IP address insertion The tests that require coreos-metadata and CLC templating could not run because for QEMU only a static replacement was done before the CLC configuration was rendered. Add support for the template variables by treating QEMU as a custom platform for the transpiler, making use of a coreos-metadata service that has the IP addresses hardcoded. --- platform/local/flight.go | 2 +- platform/machine/qemu/cluster.go | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/platform/local/flight.go b/platform/local/flight.go index 0565f0e1e..2a9013257 100644 --- a/platform/local/flight.go +++ b/platform/local/flight.go @@ -50,7 +50,7 @@ func NewLocalFlight(opts *platform.Options, platformName platform.Name) (*LocalF } nsdialer := network.NewNsDialer(nshandle) - bf, err := platform.NewBaseFlightWithDialer(opts, platformName, "", nsdialer) + bf, err := platform.NewBaseFlightWithDialer(opts, platformName, "custom", nsdialer) if err != nil { nshandle.Close() return nil, fmt.Errorf("creating new base flight failed: %v", err) diff --git a/platform/machine/qemu/cluster.go b/platform/machine/qemu/cluster.go index aa7f60111..4cfd4b0fb 100644 --- a/platform/machine/qemu/cluster.go +++ b/platform/machine/qemu/cluster.go @@ -59,8 +59,8 @@ func (qc *Cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo ip := strings.Split(netif.DHCPv4[0].String(), "/")[0] conf, err := qc.RenderUserData(userdata, map[string]string{ - "$public_ipv4": ip, - "$private_ipv4": ip, + "$public_ipv4": "${COREOS_CUSTOM_PUBLIC_IPV4}", + "$private_ipv4": "${COREOS_CUSTOM_PRIVATE_IPV4}", }) if err != nil { qc.mu.Unlock() @@ -68,6 +68,18 @@ func (qc *Cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo } qc.mu.Unlock() + conf.AddSystemdUnit("coreos-metadata.service", `[Unit] +Description=QEMU metadata agent +After=nss-lookup.target +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +Environment=OUTPUT=/run/metadata/coreos +ExecStart=/usr/bin/mkdir --parent /run/metadata +ExecStart=/usr/bin/bash -c 'echo "COREOS_CUSTOM_PRIVATE_IPV4=`+ip+`\nCOREOS_CUSTOM_PUBLIC_IPV4=`+ip+`\n" > ${OUTPUT}'`, false) + var confPath string if conf.IsIgnition() { confPath = filepath.Join(dir, "ignition.json") From b5d540e80dc448b9a4162fb31a2d8e62969a40df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20L=C3=BCke?= Date: Mon, 19 Apr 2021 14:19:45 +0200 Subject: [PATCH 2/4] platform/machine/*/cluster.go: update custom coreos-metadata service The coreos-metadata service in the Flatcar OEMs normally writes to a "flatcar" file and symlinks it. Do the same for the custom ESX and QEMU coreos-metadata services. --- platform/machine/esx/cluster.go | 6 ++++-- platform/machine/qemu/cluster.go | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/platform/machine/esx/cluster.go b/platform/machine/esx/cluster.go index 2e5bf8c8c..fcb535184 100644 --- a/platform/machine/esx/cluster.go +++ b/platform/machine/esx/cluster.go @@ -90,9 +90,11 @@ Wants=network-online.target [Service] Type=oneshot -Environment=OUTPUT=/run/metadata/coreos +Environment=OUTPUT=/run/metadata/flatcar ExecStart=/usr/bin/mkdir --parent /run/metadata -ExecStart=/usr/bin/bash -c 'echo "COREOS_CUSTOM_PRIVATE_IPV4=$(ip addr show ens192 | grep "inet 10." | grep -Po "inet \K[\d.]+")\nCOREOS_CUSTOM_PUBLIC_IPV4=$(ip addr show ens192 | grep -v "inet 10." | grep -Po "inet \K[\d.]+")" > ${OUTPUT}'`, false) +ExecStart=/usr/bin/bash -c 'echo "COREOS_CUSTOM_PRIVATE_IPV4=$(ip addr show ens192 | grep "inet 10." | grep -Po "inet \K[\d.]+")\nCOREOS_CUSTOM_PUBLIC_IPV4=$(ip addr show ens192 | grep -v "inet 10." | grep -Po "inet \K[\d.]+")" > ${OUTPUT}' +ExecStartPost=/usr/bin/ln -fs /run/metadata/flatcar /run/metadata/coreos +`, false) instance, err := ec.flight.api.CreateDevice(ec.vmname(), conf, ipPairMaybe) if err != nil { diff --git a/platform/machine/qemu/cluster.go b/platform/machine/qemu/cluster.go index 4cfd4b0fb..4859fac35 100644 --- a/platform/machine/qemu/cluster.go +++ b/platform/machine/qemu/cluster.go @@ -76,9 +76,11 @@ Wants=network-online.target [Service] Type=oneshot -Environment=OUTPUT=/run/metadata/coreos +Environment=OUTPUT=/run/metadata/flatcar ExecStart=/usr/bin/mkdir --parent /run/metadata -ExecStart=/usr/bin/bash -c 'echo "COREOS_CUSTOM_PRIVATE_IPV4=`+ip+`\nCOREOS_CUSTOM_PUBLIC_IPV4=`+ip+`\n" > ${OUTPUT}'`, false) +ExecStart=/usr/bin/bash -c 'echo "COREOS_CUSTOM_PRIVATE_IPV4=`+ip+`\nCOREOS_CUSTOM_PUBLIC_IPV4=`+ip+`\n" > ${OUTPUT}' +ExecStartPost=/usr/bin/ln -fs /run/metadata/flatcar /run/metadata/coreos +`, false) var confPath string if conf.IsIgnition() { From 92a5578e896026cde204f606efe11dbd180ccdcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20L=C3=BCke?= Date: Mon, 19 Apr 2021 13:45:41 +0200 Subject: [PATCH 3/4] platform/local/dnsmasq: hardcode DNS server for QEMU tests The system may have 127.0.0.53 as DNS server which is the systemd-resolved stub. This is no reachable from the dnsmasq's network namespace which has its own loopback interface. Hardcode the DNS servers to Cloudflare's and Google's to ensure DNS is working regardless of the host /etc/resolv.conf file. --- platform/local/dnsmasq.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/platform/local/dnsmasq.go b/platform/local/dnsmasq.go index e0d7a0362..64e906311 100644 --- a/platform/local/dnsmasq.go +++ b/platform/local/dnsmasq.go @@ -89,6 +89,11 @@ leasefile-ro log-facility=- pid-file= +# hardcode DNS servers to avoid using systemd-resolved on the unreachable 127.0.0.53 +dhcp-option=6,1.1.1.1,1.0.0.1,8.8.8.8 +no-resolv +no-hosts + enable-ra # point NTP at this host (0.0.0.0 and :: are special) From a6ae6ff86210a09c57717c6c935303589fb4bb84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20L=C3=BCke?= Date: Mon, 19 Apr 2021 14:16:23 +0200 Subject: [PATCH 4/4] kola/tests/*: enable more tests for QEMU --- kola/tests/etcd/discovery.go | 6 +++--- kola/tests/flannel/flannel.go | 4 ++-- kola/tests/ignition/security.go | 2 +- kola/tests/locksmith/locksmith.go | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/kola/tests/etcd/discovery.go b/kola/tests/etcd/discovery.go index 5cd7752bb..f0a316ad0 100644 --- a/kola/tests/etcd/discovery.go +++ b/kola/tests/etcd/discovery.go @@ -38,7 +38,7 @@ func init() { initial_advertise_peer_urls: http://{PRIVATE_IPV4}:2380 discovery: $discovery`), Distros: []string{"cl"}, - ExcludePlatforms: []string{"qemu", "qemu-unpriv"}, + ExcludePlatforms: []string{"qemu-unpriv"}, }) register.Register(®ister.Test{ @@ -54,7 +54,7 @@ etcd: initial_advertise_peer_urls: http://{PRIVATE_IPV4}:2380 discovery: $discovery `), - ExcludePlatforms: []string{"esx", "qemu", "qemu-unpriv"}, // etcd-member requires ct rendering + ExcludePlatforms: []string{"esx", "qemu-unpriv"}, // etcd-member requires ct rendering and networking Distros: []string{"cl"}, }) @@ -73,7 +73,7 @@ etcd: initial_advertise_peer_urls: http://127.0.0.1:2380 `), Distros: []string{"cl"}, - ExcludePlatforms: []string{"qemu", "qemu-unpriv"}, + ExcludePlatforms: []string{"qemu-unpriv"}, }) } diff --git a/kola/tests/flannel/flannel.go b/kola/tests/flannel/flannel.go index fa09eed60..c69ad4d08 100644 --- a/kola/tests/flannel/flannel.go +++ b/kola/tests/flannel/flannel.go @@ -63,7 +63,7 @@ func init() { ClusterSize: 3, Name: "cl.flannel.udp", Distros: []string{"cl"}, - ExcludePlatforms: []string{"qemu", "qemu-unpriv"}, + ExcludePlatforms: []string{"qemu-unpriv"}, UserData: flannelConf.Subst("$type", "udp"), }) @@ -72,7 +72,7 @@ func init() { ClusterSize: 3, Name: "cl.flannel.vxlan", Distros: []string{"cl"}, - ExcludePlatforms: []string{"qemu", "qemu-unpriv"}, + ExcludePlatforms: []string{"qemu-unpriv"}, UserData: flannelConf.Subst("$type", "vxlan"), }) } diff --git a/kola/tests/ignition/security.go b/kola/tests/ignition/security.go index 37eb882de..918a9f763 100644 --- a/kola/tests/ignition/security.go +++ b/kola/tests/ignition/security.go @@ -80,7 +80,7 @@ func init() { // ESX: Currently Ignition does not support static IPs during the initramfs // DO: https://github.com/coreos/bugs/issues/2205 // Packet & QEMU: https://github.com/coreos/ignition/issues/645 - ExcludePlatforms: []string{"esx", "do", "packet", "qemu"}, + ExcludePlatforms: []string{"esx", "do", "packet"}, Distros: []string{"cl", "fcos", "rhcos"}, }) } diff --git a/kola/tests/locksmith/locksmith.go b/kola/tests/locksmith/locksmith.go index 9824d949d..37da4d157 100644 --- a/kola/tests/locksmith/locksmith.go +++ b/kola/tests/locksmith/locksmith.go @@ -46,7 +46,7 @@ etcd: listen_peer_urls: http://{PRIVATE_IPV4}:2380 discovery: $discovery`), Distros: []string{"cl"}, - ExcludePlatforms: []string{"qemu", "qemu-unpriv"}, + ExcludePlatforms: []string{"qemu-unpriv"}, }) register.Register(®ister.Test{ Name: "coreos.locksmith.reboot",