Merge pull request #282 from webern/models

stop caring about generated code and variants
bottlerocket-os · Jun 13, 2024 · 719137a · 719137a
2 parents 0e48961 + 33185dc
commit 719137a
Show file tree

Hide file tree

Showing 7 changed files with 50 additions and 240 deletions.
diff --git a/local/alpha-sdk.Dockerfile b/local/alpha-sdk.Dockerfile
@@ -12,10 +12,6 @@ COPY build/rpms/*/*.rpm /twoliter/alpha/build/rpms/
 COPY .cargo/sbkeys/generate-local-sbkeys /twoliter/alpha/sbkeys/generate-local-sbkeys
 COPY .cargo/sbkeys/generate-aws-sbkeys /twoliter/alpha/sbkeys/generate-aws-sbkeys
 
-# These directories are needed by the build system's Dockerfile. To be eliminated later.
-COPY sources/logdog/conf/current /twoliter/alpha/sources/logdog/conf/current
-COPY sources/models/src/variant /twoliter/alpha/sources/models/src/variant
-
 # TODO - move these to an RPM package so we don't need to copy them here.
 COPY LICENSE-APACHE /twoliter/alpha/licenses/LICENSE-APACHE
 COPY LICENSE-MIT /twoliter/alpha/licenses/LICENSE-MIT

diff --git a/tests/projects/local-kit/sources/models/src/variant/.keep b/tests/projects/local-kit/sources/models/src/variant/.keep
diff --git a/tools/buildsys/src/args.rs b/tools/buildsys/src/args.rs
@@ -101,21 +101,6 @@ pub(crate) struct BuildPackageArgs {
     #[arg(long, env = "BUILDSYS_PACKAGES_DIR")]
     pub(crate) packages_dir: PathBuf,
 
-    #[arg(long, env = "BUILDSYS_VARIANT")]
-    pub(crate) variant: String,
-
-    #[arg(long, env = "BUILDSYS_VARIANT_PLATFORM")]
-    pub(crate) variant_platform: String,
-
-    #[arg(long, env = "BUILDSYS_VARIANT_RUNTIME")]
-    pub(crate) variant_runtime: String,
-
-    #[arg(long, env = "BUILDSYS_VARIANT_FAMILY")]
-    pub(crate) variant_family: String,
-
-    #[arg(long, env = "BUILDSYS_VARIANT_FLAVOR")]
-    pub(crate) variant_flavor: String,
-
     /// version_build is used along with version_build_timestamp in setting the Release of a Package. The Release is
     /// set in the form "<timestamp of latest project commit>.<latest project commit short sha>.br1" in RPMs.
     /// The value defaults to the latest commit of a project.

diff --git a/tools/buildsys/src/builder.rs b/tools/buildsys/src/builder.rs
@@ -127,21 +127,10 @@ impl CommonBuildArgs {
 }
 
 struct PackageBuildArgs {
-    /// The package might need to know what the `image_features` are going to be for the variant
-    /// it is going to be used in downstream. This is because certain packages will be built
-    /// differently based on certain image features such as cgroupsv1 vs cgroupsv2. During a
-    /// package build, these are determined by looking at the variant's Cargo.toml file based on
-    /// what was found in `BUILDSYS_VARIANT`.
-    image_features: HashSet<ImageFeature>,
     package: String,
     package_dependencies: Vec<String>,
     kit_dependencies: Vec<String>,
     external_kit_dependencies: Vec<String>,
-    variant: String,
-    variant_family: String,
-    variant_flavor: String,
-    variant_platform: String,
-    variant_runtime: String,
     version_build: String,
     version_build_epoch: String,
     version_build_timestamp: String,
@@ -185,18 +174,9 @@ impl crate::builder::PackageBuildArgs {
         );
         args.build_arg("PACKAGE", &self.package);
         args.build_arg("PACKAGE_DEPENDENCIES", self.package_dependencies.join(" "));
-        args.build_arg("VARIANT", &self.variant);
-        args.build_arg("VARIANT_FAMILY", &self.variant_family);
-        args.build_arg("VARIANT_FLAVOR", &self.variant_flavor);
-        args.build_arg("VARIANT_PLATFORM", &self.variant_platform);
-        args.build_arg("VARIANT_RUNTIME", &self.variant_runtime);
         args.build_arg("BUILD_ID", &self.version_build);
         args.build_arg("BUILD_EPOCH", &self.version_build_epoch);
         args.build_arg("BUILD_ID_TIMESTAMP", &self.version_build_timestamp);
-        for image_feature in &self.image_features {
-            args.build_arg(format!("{}", image_feature), "1");
-        }
-
         args
     }
 }
@@ -341,11 +321,7 @@ pub(crate) struct DockerBuild {
 
 impl DockerBuild {
     /// Create a new `DockerBuild` that can build a package.
-    pub(crate) fn new_package(
-        args: BuildPackageArgs,
-        manifest: &Manifest,
-        image_features: HashSet<ImageFeature>,
-    ) -> Result<Self> {
+    pub(crate) fn new_package(args: BuildPackageArgs, manifest: &Manifest) -> Result<Self> {
         let package = manifest.info().package_name();
         let per_package_dir = format!("{}/{}", args.packages_dir.display(), package).into();
         let old_package_dir = format!("{}", args.packages_dir.display()).into();
@@ -373,18 +349,12 @@ impl DockerBuild {
                 OutputCleanup::BeforeBuild,
             ),
             target_build_args: TargetBuildArgs::Package(PackageBuildArgs {
-                image_features,
                 package: package.to_string(),
                 package_dependencies: manifest.package_dependencies().context(error::GraphSnafu)?,
                 kit_dependencies: manifest.kit_dependencies().context(error::GraphSnafu)?,
                 external_kit_dependencies: ExternalKitMetadataView::load(args.common.root_dir)
                     .context(error::GraphSnafu)?
                     .list(),
-                variant: args.variant,
-                variant_family: args.variant_family,
-                variant_flavor: args.variant_flavor,
-                variant_platform: args.variant_platform,
-                variant_runtime: args.variant_runtime,
                 version_build: args.version_build,
                 version_build_epoch: args.version_build_epoch,
                 version_build_timestamp: args.version_build_timestamp,

diff --git a/tools/buildsys/src/main.rs b/tools/buildsys/src/main.rs
@@ -19,15 +19,14 @@ use crate::args::{
     BuildKitArgs, BuildPackageArgs, BuildVariantArgs, Buildsys, Command, RepackVariantArgs,
 };
 use crate::builder::DockerBuild;
-use buildsys::manifest::{BundleModule, ImageFeature, Manifest, ManifestInfo, SupportedArch};
+use buildsys::manifest::{BundleModule, Manifest, ManifestInfo, SupportedArch};
 use buildsys_config::EXTERNAL_KIT_METADATA;
 use cache::LookasideCache;
 use clap::Parser;
 use gomod::GoMod;
 use project::ProjectInfo;
 use snafu::{ensure, ResultExt};
 use spec::SpecInfo;
-use std::collections::HashSet;
 use std::path::{Path, PathBuf};
 use std::process;
 
@@ -130,18 +129,8 @@ fn build_package(args: BuildPackageArgs) -> Result<()> {
     let manifest = Manifest::new(&manifest_path, &args.common.cargo_metadata_path)
         .context(error::ManifestParseSnafu)?;
 
-    let image_features = if std::env::var("BUILDSYS_DEPRECATED_FEATURE_VARIANT_SENSITIVITY").is_ok()
-    {
-        get_package_features_and_emit_cargo_watches_for_variant_sensitivity(
-            &manifest,
-            &args.common.root_dir,
-            &args.variant,
-            args.common.arch,
-        )?
-    } else {
-        ensure_package_is_not_variant_sensitive(&manifest, &manifest_path)?;
-        HashSet::default()
-    };
+    // Check for a deprecated key and error if it is detected.
+    ensure_package_is_not_variant_sensitive(&manifest, &manifest_path)?;
 
     if let Some(files) = manifest.info().external_files() {
         let lookaside_cache = LookasideCache::new(
@@ -198,7 +187,7 @@ fn build_package(args: BuildPackageArgs) -> Result<()> {
         println!("cargo:rerun-if-changed={}", f.display());
     }
 
-    DockerBuild::new_package(args, &manifest, image_features)
+    DockerBuild::new_package(args, &manifest)
         .context(error::BuilderInstantiationSnafu)?
         .build()
         .context(error::BuildAttemptSnafu)
@@ -274,69 +263,6 @@ fn supported_arch(manifest: &ManifestInfo, arch: SupportedArch) -> Result<()> {
     Ok(())
 }
 
-fn get_package_features_and_emit_cargo_watches_for_variant_sensitivity(
-    manifest: &Manifest,
-    root_dir: &Path,
-    variant: &str,
-    arch: SupportedArch,
-) -> Result<HashSet<ImageFeature>> {
-    let package_features = manifest.info().package_features();
-
-    // Load the Variant manifest to find image features that may affect the package build.
-    let variant_manifest_path = root_dir.join("variants").join(variant).join("Cargo.toml");
-
-    let variant_manifest =
-        ManifestInfo::new(variant_manifest_path).context(error::ManifestParseSnafu)?;
-    supported_arch(&variant_manifest, arch)?;
-    let mut image_features = variant_manifest.image_features();
-
-    // For any package feature specified in the package manifest, track the corresponding
-    // environment variable for changes to the ambient set of image features for the current
-    // variant.
-    if let Some(package_features) = &package_features {
-        for package_feature in package_features {
-            println!(
-                "cargo:rerun-if-env-changed=BUILDSYS_VARIANT_IMAGE_FEATURE_{}",
-                package_feature
-            );
-        }
-    }
-
-    // Keep only the image features that the package has indicated that it tracks, if any.
-    if let Some(image_features) = &mut image_features {
-        match package_features {
-            Some(package_features) => image_features.retain(|k| package_features.contains(k)),
-            None => image_features.clear(),
-        }
-    }
-
-    // If manifest has package.metadata.build-package.variant-sensitive set, then track the
-    // appropriate environment variable for changes.
-    if let Some(sensitivity) = manifest.info().variant_sensitive() {
-        use buildsys::manifest::{SensitivityType::*, VariantSensitivity::*};
-        fn emit_variant_env(suffix: Option<&str>) {
-            if let Some(suffix) = suffix {
-                println!(
-                    "cargo:rerun-if-env-changed=BUILDSYS_VARIANT_{}",
-                    suffix.to_uppercase()
-                );
-            } else {
-                println!("cargo:rerun-if-env-changed=BUILDSYS_VARIANT");
-            }
-        }
-        match sensitivity {
-            Any(false) => (),
-            Any(true) => emit_variant_env(None),
-            Specific(Platform) => emit_variant_env(Some("platform")),
-            Specific(Runtime) => emit_variant_env(Some("runtime")),
-            Specific(Family) => emit_variant_env(Some("family")),
-            Specific(Flavor) => emit_variant_env(Some("flavor")),
-        }
-    }
-
-    Ok(image_features.unwrap_or_default())
-}
-
 /// Prior to the release of Kits as a build feature, packages could, and did, declare themselves
 /// sensitive to various Variant features so that they could be conditionally compiled based on
 /// what variant was being built. This is no longer the case, so we enforce that these keys are no

diff --git a/twoliter/embedded/build.Dockerfile b/twoliter/embedded/build.Dockerfile
@@ -32,64 +32,6 @@ COPY --chown=1000:1000 --from=sdk /tmp /cache
 # Ensure the ARG variables are used in the layer to prevent reuse by other builds.
 COPY --chown=1000:1000 Twoliter.toml /cache/.${PACKAGE}.${ARCH}.${TOKEN}
 
-# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=
-# Some builds need to modify files in the source directory, for example Rust software using
-# build.rs to generate code.  The source directory is mounted in using "--mount=source"
-# which is owned by root, and we need to modify it as the builder user.  To get around this,
-# we can use a "cache" mount, which we just won't share or reuse.  We mount a cache into the
-# location we need to change, and in some cases, set up symlinks so that it looks like a
-# normal part of the source tree.  (This is like a tmpfs mount, but cache mounts have more
-# flexibility - you can specify a source to set them up beforehand, specify uid/gid, etc.)
-# This cache is also variant-specific (in addition to package and arch, like the one above)
-# for cases where we need to build differently per variant; the cache will be empty if you
-# change BUILDSYS_VARIANT.
-FROM scratch AS variantcache
-ARG PACKAGE
-ARG ARCH
-ARG VARIANT
-ARG TOKEN
-# We can't create directories via RUN in a scratch container, so take an existing one.
-COPY --chown=1000:1000 --from=sdk /tmp /variantcache
-# Ensure the ARG variables are used in the layer to prevent reuse by other builds.
-COPY --chown=1000:1000 Twoliter.toml /variantcache/.${PACKAGE}.${ARCH}.${VARIANT}.${TOKEN}
-
-# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=
-# Generate the expected RPM macros and bconds.
-FROM sdk as rpm-macros-and-bconds
-ARG VARIANT
-ARG VARIANT_PLATFORM
-ARG VARIANT_RUNTIME
-ARG VARIANT_FAMILY
-ARG VARIANT_FLAVOR
-ARG GRUB_SET_PRIVATE_VAR
-ARG UEFI_SECURE_BOOT
-ARG SYSTEMD_NETWORKD
-ARG UNIFIED_CGROUP_HIERARCHY
-ARG XFS_DATA_PARTITION
-ARG FIPS
-
-USER builder
-WORKDIR /home/builder
-RUN \
-   export RPM_MACROS="generated.rpmmacros" \
-   && export RPM_BCONDS="generated.bconds" \
-   && echo "%_cross_variant ${VARIANT}" > "${RPM_MACROS}" \
-   && echo "%_cross_variant_platform ${VARIANT_PLATFORM}" >> "${RPM_MACROS}" \
-   && echo "%_cross_variant_runtime ${VARIANT_RUNTIME}" >> "${RPM_MACROS}" \
-   && echo "%_cross_variant_family ${VARIANT_FAMILY}" >> "${RPM_MACROS}" \
-   && echo "%_cross_variant_flavor ${VARIANT_FLAVOR:-none}" >> "${RPM_MACROS}" \
-   && echo "%_topdir /home/builder/rpmbuild" >> "${RPM_MACROS}" \
-   && echo "%bcond_without $(V=${VARIANT_PLATFORM,,}; echo ${V//-/_})_platform" > "${RPM_BCONDS}" \
-   && echo "%bcond_without $(V=${VARIANT_RUNTIME,,}; echo ${V//-/_})_runtime" >> "${RPM_BCONDS}" \
-   && echo "%bcond_without $(V=${VARIANT_FAMILY,,}; echo ${V//-/_})_family" >> "${RPM_BCONDS}" \
-   && echo "%bcond_without $(V=${VARIANT_FLAVOR:-no}; V=${V,,}; echo ${V//-/_})_flavor" >> "${RPM_BCONDS}" \
-   && echo -e -n "${GRUB_SET_PRIVATE_VAR:+%bcond_without grub_set_private_var\n}" >> "${RPM_BCONDS}" \
-   && echo -e -n "${FIPS:+%bcond_without fips\n}" >> "${RPM_BCONDS}" \
-   && echo -e -n "${UEFI_SECURE_BOOT:+%bcond_without uefi_secure_boot\n}" >> "${RPM_BCONDS}" \
-   && echo -e -n "${SYSTEMD_NETWORKD:+%bcond_without systemd_networkd\n}" >> "${RPM_BCONDS}" \
-   && echo -e -n "${UNIFIED_CGROUP_HIERARCHY:+%bcond_without unified_cgroup_hierarchy\n}" >> "${RPM_BCONDS}" \
-   && echo -e -n "${XFS_DATA_PARTITION:+%bcond_without xfs_data_partition\n}" >> "${RPM_BCONDS}"
-
 # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=
 # Builds an RPM package from a spec file.
 FROM sdk AS rpmbuild
@@ -99,14 +41,10 @@ ARG KIT_DEPENDENCIES
 ARG EXTERNAL_KIT_DEPENDENCIES
 ARG ARCH
 ARG NOCACHE
-ARG VARIANT
-ARG SYSTEMD_NETWORKD
 ARG BUILD_ID
 ARG BUILD_ID_TIMESTAMP
 ARG BUILD_EPOCH
-ENV SYSTEMD_NETWORKD=${SYSTEMD_NETWORKD}
-ENV VARIANT=${VARIANT}
-ENV BUILD_ID=${BUILD_ID} 
+ENV BUILD_ID=${BUILD_ID}
 ENV BUILD_ID_TIMESTAMP=${BUILD_ID_TIMESTAMP}
 ENV BUILD_EPOCH=${BUILD_EPOCH}
 WORKDIR /home/builder
@@ -115,18 +53,16 @@ USER builder
 ENV PACKAGE=${PACKAGE} ARCH=${ARCH}
 COPY ./packages/${PACKAGE}/ .
 
-COPY --chown=builder --from=rpm-macros-and-bconds /home/builder/generated.* .
-
-# Merge generated bconds with the package spec, and put sources in the right place. Set Epoch values for the package 
+# Copy over the target-specific macros, and put sources in the right place. Set Epoch values for the package
 # and all of its subpackages. Additionally, for packages declaring explicit package versions requirements via 
 # "{Requires,Conflicts,Obsoletes}: = ...", ensure the epoch for the specified package is set.
 # "[Requires|Conflicts|Obsoletes]:.*[=>,>,<,<=,=]\) \(\%{version}\-\%{release}$\)" matches any line in a form like:
 # "Requires: %{name}-modules = %{version}-%{release}". The full `sed` expression below captures this match into
 # two groups and insert the Epoch value such that the result is "Requires: %{name}-modules = EPOCH:%{version}-%{release}"
 RUN \
-   cat "/usr/lib/rpm/platform/${ARCH}-bottlerocket/macros" generated.rpmmacros > .rpmmacros \
+   cp "/usr/lib/rpm/platform/${ARCH}-bottlerocket/macros" .rpmmacros \
    && echo "Epoch: ${BUILD_EPOCH}" >> rpmbuild/SPECS/${PACKAGE}.spec \
-   && cat generated.bconds ${PACKAGE}.spec >> rpmbuild/SPECS/${PACKAGE}.spec \
+   && cat ${PACKAGE}.spec >> rpmbuild/SPECS/${PACKAGE}.spec \
    && sed -i "/^%package\s.*$/a Epoch: ${BUILD_EPOCH}" rpmbuild/SPECS/${PACKAGE}.spec \
    && sed -i "s;\([Requires|Conflicts|Obsoletes]:.*[=>,>,<,<=,=]\) \(\%{version}\-\%{release}$\);\1 ${BUILD_EPOCH}:\2;" rpmbuild/SPECS/${PACKAGE}.spec \
    && find . -maxdepth 1 -not -path '*/\.*' -type f -exec mv {} rpmbuild/SOURCES/ \; \
@@ -173,12 +109,9 @@ RUN --mount=target=/host \
 # Ensure that the target binutils that `find-debuginfo.sh` uses are present in $PATH.
 ENV PATH="/usr/${ARCH}-bottlerocket-linux-gnu/debuginfo/bin:${PATH}"
 
-# We use the "nocache" writable space to generate code where necessary, like the variant-
-# specific models.
 USER builder
 RUN --mount=source=.cargo,target=/home/builder/.cargo \
     --mount=type=cache,target=/home/builder/.cache,from=cache,source=/cache \
-    --mount=type=cache,target=/home/builder/rpmbuild/BUILD/sources/models/src/variant,from=variantcache,source=/variantcache \
     --mount=source=sources,target=/home/builder/rpmbuild/BUILD/sources \
     # The dist tag is set as the `Release` field in Bottlerocket RPMs. Define it to be
     # in the form <timestamp of latest commit>.<latest commit short sha>.br1
@@ -235,9 +168,47 @@ COPY --from=kitbuild /home/builder/output/. /output/
 # Section 3: The following build stages are used to create a Bottlerocket image once all of
 # the rpm files have been created by repeatedly using Sections 1 and 2.
 
+# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=
+# Generate the expected RPM macros and bconds.
+FROM sdk as rpm-macros-and-bconds
+ARG VARIANT
+ARG VARIANT_PLATFORM
+ARG VARIANT_RUNTIME
+ARG VARIANT_FAMILY
+ARG VARIANT_FLAVOR
+ARG GRUB_SET_PRIVATE_VAR
+ARG UEFI_SECURE_BOOT
+ARG SYSTEMD_NETWORKD
+ARG UNIFIED_CGROUP_HIERARCHY
+ARG XFS_DATA_PARTITION
+ARG FIPS
+
+USER builder
+WORKDIR /home/builder
+RUN \
+   export RPM_MACROS="generated.rpmmacros" \
+   && export RPM_BCONDS="generated.bconds" \
+   && echo "%_cross_variant ${VARIANT}" > "${RPM_MACROS}" \
+   && echo "%_cross_variant_platform ${VARIANT_PLATFORM}" >> "${RPM_MACROS}" \
+   && echo "%_cross_variant_runtime ${VARIANT_RUNTIME}" >> "${RPM_MACROS}" \
+   && echo "%_cross_variant_family ${VARIANT_FAMILY}" >> "${RPM_MACROS}" \
+   && echo "%_cross_variant_flavor ${VARIANT_FLAVOR:-none}" >> "${RPM_MACROS}" \
+   && echo "%_topdir /home/builder/rpmbuild" >> "${RPM_MACROS}" \
+   && echo "%bcond_without $(V=${VARIANT_PLATFORM,,}; echo ${V//-/_})_platform" > "${RPM_BCONDS}" \
+   && echo "%bcond_without $(V=${VARIANT_RUNTIME,,}; echo ${V//-/_})_runtime" >> "${RPM_BCONDS}" \
+   && echo "%bcond_without $(V=${VARIANT_FAMILY,,}; echo ${V//-/_})_family" >> "${RPM_BCONDS}" \
+   && echo "%bcond_without $(V=${VARIANT_FLAVOR:-no}; V=${V,,}; echo ${V//-/_})_flavor" >> "${RPM_BCONDS}" \
+   && echo -e -n "${GRUB_SET_PRIVATE_VAR:+%bcond_without grub_set_private_var\n}" >> "${RPM_BCONDS}" \
+   && echo -e -n "${FIPS:+%bcond_without fips\n}" >> "${RPM_BCONDS}" \
+   && echo -e -n "${UEFI_SECURE_BOOT:+%bcond_without uefi_secure_boot\n}" >> "${RPM_BCONDS}" \
+   && echo -e -n "${SYSTEMD_NETWORKD:+%bcond_without systemd_networkd\n}" >> "${RPM_BCONDS}" \
+   && echo -e -n "${UNIFIED_CGROUP_HIERARCHY:+%bcond_without unified_cgroup_hierarchy\n}" >> "${RPM_BCONDS}" \
+   && echo -e -n "${XFS_DATA_PARTITION:+%bcond_without xfs_data_partition\n}" >> "${RPM_BCONDS}"
+
+
 # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=
 # Creates an RPM repository from packages created in Section 1 and kits from Section 2.
-FROM sdk AS repobuild
+FROM rpm-macros-and-bconds AS repobuild
 # The list of packages from the variant Cargo.toml package.metadata.build-variant.packages section.
 ARG PACKAGES
 # The complete list of non-kit packages required by way of pure package-to-package dependencies.
@@ -250,8 +221,6 @@ ARG NOCACHE
 WORKDIR /home/builder
 USER builder
 
-COPY --chown=builder --from=rpm-macros-and-bconds /home/builder/generated.* .
-
 # Build the metadata RPM for the variant.
 RUN --mount=target=/host \
    cat "/usr/lib/rpm/platform/${ARCH}-bottlerocket/macros" generated.rpmmacros > .rpmmacros \