From 0a978dd32768a4bb53c64cb715adf1674cc54125 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 11 Mar 2020 12:05:49 -0400 Subject: [PATCH] Add support for 4k metal images First, add a new `buildextend-metal4k` command to create 4k disk images. Then, teach `kola` and `cosa run` to read these images. To test: cosa run -I metal4k One potentially controversial bit here is that this requires a newer libguestfs which isn't in f31 yet, so we pull it from f32 for now. Closes: https://github.com/coreos/fedora-coreos-tracker/issues/385 --- build.sh | 10 ++++++ mantle/cmd/kola/options.go | 1 + mantle/cmd/kola/qemuexec.go | 5 +++ mantle/platform/machine/unprivqemu/cluster.go | 5 +++ mantle/platform/machine/unprivqemu/flight.go | 3 +- mantle/platform/qemu.go | 19 +++++++--- src/cmd-buildextend-metal | 36 ++++++++++++------- src/cmd-buildextend-metal4k | 1 + src/cmd-run | 10 +++++- src/coreos-assembler | 2 +- 10 files changed, 71 insertions(+), 21 deletions(-) create mode 120000 src/cmd-buildextend-metal4k diff --git a/build.sh b/build.sh index 7b77926ad7..ac55b5fee5 100755 --- a/build.sh +++ b/build.sh @@ -41,6 +41,16 @@ install_rpms() { # xargs is part of findutils, which may not be installed yum -y install /usr/bin/xargs + # We need a newer guestfish which supports 4k drives (XXX: tag into + # continuous tag?) + local libguestfs_koji="https://kojipkgs.fedoraproject.org//packages/libguestfs/1.42.0/1.fc32" + yum -y install \ + ${libguestfs_koji}/x86_64/libguestfs-1.42.0-1.fc32.x86_64.rpm \ + ${libguestfs_koji}/x86_64/libguestfs-xfs-1.42.0-1.fc32.x86_64.rpm \ + ${libguestfs_koji}/x86_64/libguestfs-tools-c-1.42.0-1.fc32.x86_64.rpm \ + ${libguestfs_koji}/noarch/libguestfs-tools-1.42.0-1.fc32.noarch.rpm \ + ${libguestfs_koji}/x86_64/perl-Sys-Guestfs-1.42.0-1.fc32.x86_64.rpm + # These are only used to build things in here. Today # we ship these in the container too to make it easier # to use the container as a development environment for itself. diff --git a/mantle/cmd/kola/options.go b/mantle/cmd/kola/options.go index e1fa516816..17df0dca60 100644 --- a/mantle/cmd/kola/options.go +++ b/mantle/cmd/kola/options.go @@ -137,6 +137,7 @@ func init() { sv(&kola.QEMUOptions.Firmware, "qemu-firmware", "bios", "Boot firmware: bios,uefi,uefi-secure") sv(&kola.QEMUOptions.DiskImage, "qemu-image", "", "path to CoreOS disk image") sv(&kola.QEMUOptions.DiskSize, "qemu-size", "", "Resize target disk via qemu-img resize [+]SIZE") + bv(&kola.QEMUOptions.Native4k, "qemu-native-4k", false, "Force 4k sectors for main disk") bv(&kola.QEMUOptions.Nvme, "qemu-nvme", false, "Use NVMe for main disk") bv(&kola.QEMUOptions.Swtpm, "qemu-swtpm", true, "Create temporary software TPM") } diff --git a/mantle/cmd/kola/qemuexec.go b/mantle/cmd/kola/qemuexec.go index 35311cffb0..6b89114a80 100644 --- a/mantle/cmd/kola/qemuexec.go +++ b/mantle/cmd/kola/qemuexec.go @@ -68,10 +68,15 @@ func runQemuExec(cmd *cobra.Command, args []string) error { if kola.QEMUOptions.Nvme { channel = "nvme" } + sectorSize := 0 + if kola.QEMUOptions.Native4k { + sectorSize = 4096 + } if err = builder.AddPrimaryDisk(&platform.Disk{ BackingFile: kola.QEMUOptions.DiskImage, Channel: channel, Size: kola.QEMUOptions.DiskSize, + SectorSize: sectorSize, }); err != nil { return errors.Wrapf(err, "failed adding primary disk") } diff --git a/mantle/platform/machine/unprivqemu/cluster.go b/mantle/platform/machine/unprivqemu/cluster.go index 163bf193b9..b155809e46 100644 --- a/mantle/platform/machine/unprivqemu/cluster.go +++ b/mantle/platform/machine/unprivqemu/cluster.go @@ -99,10 +99,15 @@ func (qc *Cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo if qc.flight.opts.Nvme { channel = "nvme" } + sectorSize := 0 + if qc.flight.opts.Native4k { + sectorSize = 4096 + } primaryDisk := platform.Disk{ BackingFile: qc.flight.opts.DiskImage, Channel: channel, Size: qc.flight.opts.DiskSize, + SectorSize: sectorSize, } if err = builder.AddPrimaryDisk(&primaryDisk); err != nil { diff --git a/mantle/platform/machine/unprivqemu/flight.go b/mantle/platform/machine/unprivqemu/flight.go index 87876c14fc..32fc2582fb 100644 --- a/mantle/platform/machine/unprivqemu/flight.go +++ b/mantle/platform/machine/unprivqemu/flight.go @@ -35,7 +35,8 @@ type Options struct { ForceConfigInjection bool - Nvme bool + Native4k bool + Nvme bool //Option to create a temporary software TPM - true by default Swtpm bool diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index f34ba306a2..5a59fc08ed 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -40,6 +40,7 @@ type Disk struct { BackingFile string // raw disk image to use. Incompatible with Size. Channel string // virtio (default), nvme DeviceOpts []string // extra options to pass to qemu. "serial=XXXX" makes disks show up as /dev/disk/by-id/virtio- + SectorSize int // if not 0, override disk sector size } type QemuInstance struct { @@ -289,11 +290,16 @@ type coreosGuestfish struct { remote string } -func newGuestfish(diskImagePath string) (*coreosGuestfish, error) { +func newGuestfish(diskImagePath string, diskSectorSize int) (*coreosGuestfish, error) { // Set guestfish backend to direct in order to avoid libvirt as backend. // Using libvirt can lead to permission denied issues if it does not have access // rights to the qcow image - cmd := exec.Command("guestfish", "--listen", "-a", diskImagePath) + guestfish_args := []string{"--listen"} + if diskSectorSize != 0 { + guestfish_args = append(guestfish_args, fmt.Sprintf("--blocksize=%d", diskSectorSize)) + } + guestfish_args = append(guestfish_args, "-a", diskImagePath) + cmd := exec.Command("guestfish", guestfish_args...) cmd.Env = append(os.Environ(), "LIBGUESTFS_BACKEND=direct") // make sure it inherits stderr so we see any error message cmd.Stderr = os.Stderr @@ -352,8 +358,8 @@ func (gf *coreosGuestfish) destroy() { // setupIgnition copies the ignition file inside the disk image and/or sets // networking kernel arguments -func setupIgnition(confPath string, knetargs string, diskImagePath string) error { - gf, err := newGuestfish(diskImagePath) +func setupIgnition(confPath string, knetargs string, diskImagePath string, diskSectorSize int) error { + gf, err := newGuestfish(diskImagePath, diskSectorSize) if err != nil { return err } @@ -438,7 +444,7 @@ func (builder *QemuBuilder) addDiskImpl(disk *Disk, primary bool) error { // requested, inject via libguestfs on the primary disk. requiresInjection := builder.Config != "" && (builder.ForceConfigInjection || !builder.supportsFwCfg()) if requiresInjection || builder.IgnitionNetworkKargs != "" { - if err = setupIgnition(builder.Config, builder.IgnitionNetworkKargs, dstFileName); err != nil { + if err = setupIgnition(builder.Config, builder.IgnitionNetworkKargs, dstFileName, disk.SectorSize); err != nil { return errors.Wrapf(err, "ignition injection with guestfs failed") } } @@ -466,6 +472,9 @@ func (builder *QemuBuilder) addDiskImpl(disk *Disk, primary bool) error { if channel == "" { channel = "virtio" } + if disk.SectorSize != 0 { + diskOpts = append(diskOpts, fmt.Sprintf("physical_block_size=%[1]d,logical_block_size=%[1]d", disk.SectorSize)) + } builder.addQcow2DiskFd(fd, channel, diskOpts) return nil } diff --git a/src/cmd-buildextend-metal b/src/cmd-buildextend-metal index 8f8d3e9d29..e6131b5412 100755 --- a/src/cmd-buildextend-metal +++ b/src/cmd-buildextend-metal @@ -9,6 +9,7 @@ dn=$(dirname "$0") # image (qemu). `buildextend-qemu` is a symlink to `buildextend-metal`. case "$(basename "$0")" in "cmd-buildextend-metal") image_type=metal;; + "cmd-buildextend-metal4k") image_type=metal4k;; "cmd-buildextend-dasd") image_type=dasd;; "cmd-buildextend-qemu") image_type=qemu;; *) fatal "called as unexpected name $0";; @@ -137,9 +138,8 @@ img=${name}-${build}-${image_type}.${basearch}.${image_format} path=${PWD}/${img} ignition_platform_id="${image_type}" -# dasd is a different disk format, but it's still metal. Just like -# if in the future we introduce a 4k sector size x86_64 image type (metal-4k). -if [ "${image_type}" = dasd ]; then +# dasd and metal4k are different disk formats, but they're still metal +if [ "${image_type}" = dasd ] || [ "${image_type}" = metal4k ]; then ignition_platform_id=metal fi @@ -171,13 +171,17 @@ echo "Disk size estimated to ${image_size}" # For bare metal and dasd images, we use the estimated image size. For IaaS/virt, we get it from # image.yaml because we want a "default" disk size that has some free space. -if [[ "${image_type}" = metal || "${image_type}" = dasd ]]; then - # Unset the root size, which will inherit from the image size - rootfs_size=0 -else - image_size="$(python3 -c 'import sys, yaml; print(yaml.safe_load(sys.stdin)["size"])' < "$configdir/image.yaml")G" - rootfs_size="${rootfs_size}M" -fi +case "${image_type}" in + metal*|dasd) + # Unset the root size, which will inherit from the image size + rootfs_size=0 + ;; + qemu) + image_size="$(python3 -c 'import sys, yaml; print(yaml.safe_load(sys.stdin)["size"])' < "$configdir/image.yaml")G" + rootfs_size="${rootfs_size}M" + ;; + *) fatal "unreachable image_type ${image_type}";; +esac disk_args=() @@ -225,12 +229,18 @@ ref_arg=${ref} if [ -n "${ref_is_temp}" ]; then ref_arg=${commit} fi + target_drive=("-drive" "if=virtio,id=target,format=${image_format},file=${path}.tmp,cache=unsafe") -if [[ $image_format == raw && $image_type == dasd ]]; then +# we need 4096 block size for ECKD DASD and (obviously) metal4k +if [[ $image_type == dasd || $image_type == metal4k ]]; then + device_type=virtio-blk + if [[ $image_type == dasd ]]; then + device_type=virtio-blk-ccw + fi target_drive=("-drive" "if=none,id=target,format=${image_format},file=${path}.tmp,cache=unsafe" \ - # we need 4096 block size for ECKD DASD - "-device" "virtio-blk-ccw,drive=target,physical_block_size=4096,logical_block_size=4096,scsi=off") + "-device" "${device_type},drive=target,physical_block_size=4096,logical_block_size=4096,scsi=off") fi + runvm "${target_drive[@]}" -- \ /usr/lib/coreos-assembler/create_disk.sh \ --disk /dev/vda \ diff --git a/src/cmd-buildextend-metal4k b/src/cmd-buildextend-metal4k new file mode 120000 index 0000000000..ad07b13c46 --- /dev/null +++ b/src/cmd-buildextend-metal4k @@ -0,0 +1 @@ +cmd-buildextend-metal \ No newline at end of file diff --git a/src/cmd-run b/src/cmd-run index be0f471b66..6c0cd9e23b 100755 --- a/src/cmd-run +++ b/src/cmd-run @@ -26,7 +26,7 @@ FIRMWARE=bios USAGE="Usage: $0 [-d /path/to/disk.qcow2] [--] [qemu options...] Options: -b --buildid Target buildid (default latest) - -I --imgtype Target image type (qemu, metal, etc. Default qemu) + -I --imgtype Target image type (qemu, metal, metal4k, etc. Default qemu) -d DISK Root disk drive (won't be changed by default) --disk-channel TYPE Communication mechanism for root device: virtio, nvme -i FILE File containing an Ignition config to merge into the default config @@ -259,6 +259,14 @@ case "${DISK_CHANNEL}" in *) die "Invalid --disk-channel ${DISK_CHANNEL}" ;; esac +if [ "${IMAGE_TYPE}" == metal4k ]; then + kola_args+=("--qemu-native-4k") + # native 4k requires a UEFI bootloader + if [ "${FIRMWARE}" == bios ]; then + FIRMWARE=uefi + fi +fi + case "${FIRMWARE}" in bios) ;; *) kola_args+=("--qemu-firmware=${FIRMWARE}") diff --git a/src/coreos-assembler b/src/coreos-assembler index 3e8fd7c88d..b9d23eb086 100755 --- a/src/coreos-assembler +++ b/src/coreos-assembler @@ -43,7 +43,7 @@ cmd=${1:-} build_commands="init fetch build run prune clean list" # commands more likely to be used in a prod pipeline only advanced_build_commands="buildprep buildupload oscontainer" -buildextend_commands="aws azure gcp ibmcloud installer live metal openstack qemu vmware vultr exoscale" +buildextend_commands="aws azure gcp ibmcloud installer live metal metal4k openstack qemu vmware vultr exoscale" utility_commands="tag sign compress koji-upload kola aws-replicate remote-prune" other_commands="shell meta" if [ -z "${cmd}" ]; then