live-generator: Rework to avoid tmpfs and overlayfs

OK many things rolled into one commit because I just
needed to test them together.  The high level goal is SELinux
enforcing for RHCOS and to mitigate a boot time race (still
not debugged).

The SELinux enforcing one turned up a huge mess because as
it happens SELinux will nuke all labels on `tmpfs`.  Which
is very problematic in general (FCOS included) but FCOS was
papering over this problem by using the systemd `relabel-extra.d`
bits.

But we don't really want to use that anymore IMO - we should
have a clean model where files are always labeled correctly in the initramfs
before we switch root.  Anything else going to lead to pain.

In order to work around the `tmpfs` bug, instead make a loopback-mounted
`xfs` filesystem (on `tmpfs`).

When the kernel is fixed to retain labels in `tmpfs` we can drop that
hack.

overlay.d/05core/usr/lib/dracut/modules.d/20live/live-generator

@@ -78,14 +78,22 @@ else
     mkdir -p /run/media/iso
     isosrc=dev/disk/by-label/${isoroot}
     isosrc_escaped=$(systemd-escape -p --suffix=device "${isosrc}")
+    initrd_rootdev_target_d="${UNIT_DIR}"/initrd-root-device.target.d
+    mkdir -p "${initrd_rootdev_target_d}"
+    cat > "${initrd_rootdev_target_d}/50-root-device.conf" <<EOF
+[Unit]
+After=${isosrc_escaped}
+Requires=${isosrc_escaped}
+EOF
     cat >"${UNIT_DIR}/run-media-iso.mount" <<EOF
 # Automatically generated by live-generator
 
 [Unit]
 DefaultDependencies=false
+After=initrd-root-device.target
+# HACK for https://github.com/coreos/fedora-coreos-config/issues/437
+After=systemd-udev-settle.service
 Before=initrd-root-fs.target
-After=${isosrc_escaped}
-Requires=${isosrc_escaped}
 
 [Mount]
 What=/${isosrc}
@@ -100,7 +108,7 @@ EOF
 [Unit]
 DefaultDependencies=false
 Before=initrd-root-fs.target
-Requires=run-media-iso.mount
+RequiresMountsFor=/run/media/iso
 
 [Mount]
 What=/run/media/iso/root.squashfs
@@ -109,67 +117,105 @@ Type=squashfs
 EOF
 fi
 
-common_etcvar_unit() {
-    cat << EOF
-# Automatically generated by live-generator
+# It turns out that `tmpfs` currently munches all SELinux labels
+# we set before policy is loaded, so we make an XFS filesystem
+# loopback mounted that's sized the same as RAM.
+# https://github.com/coreos/fedora-coreos-config/pull/499
+cat >"${UNIT_DIR}/sysroot-xfs-ephemeral-mkfs.service" <<'EOF'
 [Unit]
 DefaultDependencies=false
+# Let's be sure we have basic devices, but other than that we
+# can run really early.
+After=systemd-tmpfiles-setup-dev.service
+ConditionPathExists=/usr/lib/initrd-release
+# Something seems to be causing us to rerun?
+ConditionPathExists=!/run/ephemeral
 
-# Make sure /sysroot is mounted first, since we're mounting under there
-Requires=initrd-root-fs.target
-After=initrd-root-fs.target
-
-# Make sure our tmpfs is available
-RequiresMountsFor=/writable
-
-# Need to do this before Ignition mounts any other filesystems (potentially
-# shadowing our own mount).
-Before=ignition-mount.service
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/bin/sh -c 'set -euo pipefail; mem=$$(grep "^MemTotal" /proc/meminfo | sed -e "s,^.*: *\\([0-9]*\\) .*,\\1,") && /bin/truncate -s $${mem}k /run/ephemeral.xfsloop'
+ExecStart=/sbin/mkfs.xfs /run/ephemeral.xfsloop
+ExecStart=/bin/mkdir /run/ephemeral
 EOF
-}
-
-supports_squashfs_overlayfs=1
-case "$(uname -r)" in
-    4.18.*) supports_squashfs_overlayfs=0
-esac
+add_requires sysroot-xfs-ephemeral-mkfs.service initrd-root-fs.target
 
-if [ "${supports_squashfs_overlayfs}" = 1 ]; then
-    common_etcvar_unit > "${UNIT_DIR}/sysroot-etc.mount"
-    cat >>"${UNIT_DIR}/sysroot-etc.mount" <<EOF
+cat >>"${UNIT_DIR}/run-ephemeral.mount" <<EOF
+[Unit]
+DefaultDependencies=false
+Requires=sysroot-xfs-ephemeral-mkfs.service
+After=sysroot-xfs-ephemeral-mkfs.service
 [Mount]
-What=overlay
-Where=/sysroot/etc
-Type=overlay
-Options=lowerdir=/sysroot/etc,upperdir=/writable/etc/upper,workdir=/writable/etc/work,redirect_dir=on,index=on,xino=on
+What=/run/ephemeral.xfsloop
+Where=/run/ephemeral
+Type=xfs
+Options=loop
 EOF
-else
-    # RHEL8 can't do overlayfs on squashfs, so we just copy
-    # /etc fully into RAM.  It's not a large amount of data.
-    common_etcvar_unit > "${UNIT_DIR}/sysroot-etc-copy.service"
-    cat >>"${UNIT_DIR}/sysroot-etc-copy.service" <<EOF
+
+cat >"${UNIT_DIR}/sysroot-xfs-ephemeral-setup.service" <<EOF
+[Unit]
+DefaultDependencies=false
+RequiresMountsFor=/run/ephemeral
+ConditionPathExists=/usr/lib/initrd-release
+ConditionPathExists=!/run/ephemeral/var
+# Make sure /sysroot is mounted first, since we're mounting under there
+Requires=sysroot.mount
+After=sysroot.mount
+# And after OSTree has set up the chroot() equivalent
+After=ostree-prepare-root.service
+
+# We're part of assembling the root fs
+Before=initrd-root-fs.target
+
 [Service]
 Type=oneshot
 RemainAfterExit=yes
-ExecStart=/bin/cp -a /sysroot/etc /writable/etc-copy
+ExecStart=/bin/cp -a /sysroot/etc /run/ephemeral/etc
+ExecStart=/bin/mkdir /run/ephemeral/var
+EOF
+
+common_etcvar_unit() {
+    cat << EOF
+# Automatically generated by live-generator
+[Unit]
+DefaultDependencies=false
+# Make sure our tmpfs is available
+Requires=sysroot-xfs-ephemeral-setup.service
+After=sysroot-xfs-ephemeral-setup.service
 EOF
+}
 
-    common_etcvar_unit > "${UNIT_DIR}/sysroot-etc.mount"
+common_etcvar_unit > "${UNIT_DIR}/sysroot-etc.mount"
 cat >>"${UNIT_DIR}/sysroot-etc.mount" <<EOF
-After=sysroot-etc-copy.service
-Requires=sysroot-etc-copy.service
 [Mount]
-What=/writable/etc-copy
+What=/run/ephemeral/etc
 Where=/sysroot/etc
 Type=none
 Options=bind
 EOF
-fi
+add_requires sysroot-etc.mount initrd-root-fs.target
 
 common_etcvar_unit >"${UNIT_DIR}/sysroot-var.mount"
 cat >>"${UNIT_DIR}/sysroot-var.mount" <<EOF
 [Mount]
-What=/writable/var
+What=/run/ephemeral/var
 Where=/sysroot/var
 Type=none
 Options=bind
 EOF
+add_requires sysroot-var.mount initrd-root-fs.target
+
+cat >>"${UNIT_DIR}/sysroot-relabel.service" <<EOF
+[Unit]
+DefaultDependencies=false
+RequiresMountsFor=/sysroot/etc /sysroot/var
+Before=initrd-root-fs.target
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+# We don't need the full relabeling spam by default for these
+StandardOutput=null
+ExecStart=/bin/coreos-relabel /etc
+ExecStart=/bin/coreos-relabel /var
+EOF
+add_requires sysroot-relabel.service initrd-root-fs.target

overlay.d/05core/usr/lib/dracut/modules.d/20live/module-setup.sh

@@ -7,6 +7,8 @@ install_and_enable_unit() {
 }
 
 install() {
+    inst_multiple truncate
+
     inst_script "$moddir/is-live-image.sh" \
         "/usr/bin/is-live-image"
 
@@ -16,9 +18,6 @@ install() {
     inst_simple "$moddir/live-generator" \
         "$systemdutildir/system-generators/live-generator"
 
-    inst_simple "$moddir/coreos-populate-writable.service" \
-        "$systemdsystemunitdir/coreos-populate-writable.service"
-
     inst_simple "$moddir/coreos-live-unmount-tmpfs-var.sh" \
         "/usr/sbin/coreos-live-unmount-tmpfs-var"
 
@@ -31,9 +30,6 @@ install() {
     install_and_enable_unit "coreos-live-persist-osmet.service" \
         "default.target"
 
-    inst_simple "$moddir/writable.mount" \
-        "$systemdsystemunitdir/writable.mount"
-
     inst_simple "$moddir/coreos-liveiso-network-kargs.sh" \
         "/usr/sbin/coreos-liveiso-network-kargs"