Skip to content

Commit

Permalink
buildsys: remove package variant sensitivity
Browse files Browse the repository at this point in the history 
Remove the logic by which packages are differentially built based on
which variant they are for.
Requires bottlerocket-os/bottlerocket#4038
  • Loading branch information
@webern
webern committed Jun 12, 2024
1 parent 1b4d7e2 commit 2721a63
Show file tree
Hide file tree
Showing 4 changed files with 48 additions and 175 deletions.
15 changes: 0 additions & 15 deletions tools/buildsys/src/args.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,21 +101,6 @@ pub(crate) struct BuildPackageArgs {
#[arg(long, env = "BUILDSYS_PACKAGES_DIR")]
pub(crate) packages_dir: PathBuf,

#[arg(long, env = "BUILDSYS_VARIANT")]
pub(crate) variant: String,

#[arg(long, env = "BUILDSYS_VARIANT_PLATFORM")]
pub(crate) variant_platform: String,

#[arg(long, env = "BUILDSYS_VARIANT_RUNTIME")]
pub(crate) variant_runtime: String,

#[arg(long, env = "BUILDSYS_VARIANT_FAMILY")]
pub(crate) variant_family: String,

#[arg(long, env = "BUILDSYS_VARIANT_FLAVOR")]
pub(crate) variant_flavor: String,

/// version_build is used along with version_build_timestamp in setting the Release of a Package. The Release is
/// set in the form "<timestamp of latest project commit>.<latest project commit short sha>.br1" in RPMs.
/// The value defaults to the latest commit of a project.
Expand Down
32 changes: 1 addition & 31 deletions tools/buildsys/src/builder.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,20 +126,9 @@ impl CommonBuildArgs {
}

struct PackageBuildArgs {
/// The package might need to know what the `image_features` are going to be for the variant
/// it is going to be used in downstream. This is because certain packages will be built
/// differently based on certain image features such as cgroupsv1 vs cgroupsv2. During a
/// package build, these are determined by looking at the variant's Cargo.toml file based on
/// what was found in `BUILDSYS_VARIANT`.
image_features: HashSet<ImageFeature>,
package: String,
package_dependencies: Vec<String>,
kit_dependencies: Vec<String>,
variant: String,
variant_family: String,
variant_flavor: String,
variant_platform: String,
variant_runtime: String,
version_build: String,
version_build_epoch: String,
version_build_timestamp: String,
Expand Down Expand Up @@ -179,18 +168,9 @@ impl crate::builder::PackageBuildArgs {
args.build_arg("KIT_DEPENDENCIES", self.kit_dependencies.join(" "));
args.build_arg("PACKAGE", &self.package);
args.build_arg("PACKAGE_DEPENDENCIES", self.package_dependencies.join(" "));
args.build_arg("VARIANT", &self.variant);
args.build_arg("VARIANT_FAMILY", &self.variant_family);
args.build_arg("VARIANT_FLAVOR", &self.variant_flavor);
args.build_arg("VARIANT_PLATFORM", &self.variant_platform);
args.build_arg("VARIANT_RUNTIME", &self.variant_runtime);
args.build_arg("BUILD_ID", &self.version_build);
args.build_arg("BUILD_EPOCH", &self.version_build_epoch);
args.build_arg("BUILD_ID_TIMESTAMP", &self.version_build_timestamp);
for image_feature in &self.image_features {
args.build_arg(format!("{}", image_feature), "1");
}

args
}
}
Expand Down Expand Up @@ -330,11 +310,7 @@ pub(crate) struct DockerBuild {

impl DockerBuild {
/// Create a new `DockerBuild` that can build a package.
pub(crate) fn new_package(
args: BuildPackageArgs,
manifest: &Manifest,
image_features: HashSet<ImageFeature>,
) -> Result<Self> {
pub(crate) fn new_package(args: BuildPackageArgs, manifest: &Manifest) -> Result<Self> {
let package = manifest.info().package_name();
let per_package_dir = format!("{}/{}", args.packages_dir.display(), package).into();
let old_package_dir = format!("{}", args.packages_dir.display()).into();
Expand Down Expand Up @@ -362,15 +338,9 @@ impl DockerBuild {
OutputCleanup::BeforeBuild,
),
target_build_args: TargetBuildArgs::Package(PackageBuildArgs {
image_features,
package: package.to_string(),
package_dependencies: manifest.package_dependencies().context(error::GraphSnafu)?,
kit_dependencies: manifest.kit_dependencies().context(error::GraphSnafu)?,
variant: args.variant,
variant_family: args.variant_family,
variant_flavor: args.variant_flavor,
variant_platform: args.variant_platform,
variant_runtime: args.variant_runtime,
version_build: args.version_build,
version_build_epoch: args.version_build_epoch,
version_build_timestamp: args.version_build_timestamp,
Expand Down
82 changes: 4 additions & 78 deletions tools/buildsys/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,14 @@ use crate::args::{
BuildKitArgs, BuildPackageArgs, BuildVariantArgs, Buildsys, Command, RepackVariantArgs,
};
use crate::builder::DockerBuild;
use buildsys::manifest::{BundleModule, ImageFeature, Manifest, ManifestInfo, SupportedArch};
use buildsys::manifest::{BundleModule, Manifest, ManifestInfo, SupportedArch};
use buildsys_config::EXTERNAL_KIT_METADATA;
use cache::LookasideCache;
use clap::Parser;
use gomod::GoMod;
use project::ProjectInfo;
use snafu::{ensure, ResultExt};
use spec::SpecInfo;
use std::collections::HashSet;
use std::path::{Path, PathBuf};
use std::process;

Expand Down Expand Up @@ -130,18 +129,8 @@ fn build_package(args: BuildPackageArgs) -> Result<()> {
let manifest = Manifest::new(&manifest_path, &args.common.cargo_metadata_path)
.context(error::ManifestParseSnafu)?;

let image_features = if std::env::var("BUILDSYS_DEPRECATED_FEATURE_VARIANT_SENSITIVITY").is_ok()
{
get_package_features_and_emit_cargo_watches_for_variant_sensitivity(
&manifest,
&args.common.root_dir,
&args.variant,
args.common.arch,
)?
} else {
ensure_package_is_not_variant_sensitive(&manifest, &manifest_path)?;
HashSet::default()
};
// Check for a deprecated key and error if it is detected.
ensure_package_is_not_variant_sensitive(&manifest, &manifest_path)?;

if let Some(files) = manifest.info().external_files() {
let lookaside_cache = LookasideCache::new(
Expand Down Expand Up @@ -198,7 +187,7 @@ fn build_package(args: BuildPackageArgs) -> Result<()> {
println!("cargo:rerun-if-changed={}", f.display());
}

DockerBuild::new_package(args, &manifest, image_features)
DockerBuild::new_package(args, &manifest)
.context(error::BuilderInstantiationSnafu)?
.build()
.context(error::BuildAttemptSnafu)
Expand Down Expand Up @@ -274,69 +263,6 @@ fn supported_arch(manifest: &ManifestInfo, arch: SupportedArch) -> Result<()> {
Ok(())
}

fn get_package_features_and_emit_cargo_watches_for_variant_sensitivity(
manifest: &Manifest,
root_dir: &Path,
variant: &str,
arch: SupportedArch,
) -> Result<HashSet<ImageFeature>> {
let package_features = manifest.info().package_features();

// Load the Variant manifest to find image features that may affect the package build.
let variant_manifest_path = root_dir.join("variants").join(variant).join("Cargo.toml");

let variant_manifest =
ManifestInfo::new(variant_manifest_path).context(error::ManifestParseSnafu)?;
supported_arch(&variant_manifest, arch)?;
let mut image_features = variant_manifest.image_features();

// For any package feature specified in the package manifest, track the corresponding
// environment variable for changes to the ambient set of image features for the current
// variant.
if let Some(package_features) = &package_features {
for package_feature in package_features {
println!(
"cargo:rerun-if-env-changed=BUILDSYS_VARIANT_IMAGE_FEATURE_{}",
package_feature
);
}
}

// Keep only the image features that the package has indicated that it tracks, if any.
if let Some(image_features) = &mut image_features {
match package_features {
Some(package_features) => image_features.retain(|k| package_features.contains(k)),
None => image_features.clear(),
}
}

// If manifest has package.metadata.build-package.variant-sensitive set, then track the
// appropriate environment variable for changes.
if let Some(sensitivity) = manifest.info().variant_sensitive() {
use buildsys::manifest::{SensitivityType::*, VariantSensitivity::*};
fn emit_variant_env(suffix: Option<&str>) {
if let Some(suffix) = suffix {
println!(
"cargo:rerun-if-env-changed=BUILDSYS_VARIANT_{}",
suffix.to_uppercase()
);
} else {
println!("cargo:rerun-if-env-changed=BUILDSYS_VARIANT");
}
}
match sensitivity {
Any(false) => (),
Any(true) => emit_variant_env(None),
Specific(Platform) => emit_variant_env(Some("platform")),
Specific(Runtime) => emit_variant_env(Some("runtime")),
Specific(Family) => emit_variant_env(Some("family")),
Specific(Flavor) => emit_variant_env(Some("flavor")),
}
}

Ok(image_features.unwrap_or_default())
}

/// Prior to the release of Kits as a build feature, packages could, and did, declare themselves
/// sensitive to various Variant features so that they could be conditionally compiled based on
/// what variant was being built. This is no longer the case, so we enforce that these keys are no
Expand Down
94 changes: 43 additions & 51 deletions twoliter/embedded/build.Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,43 +32,6 @@ COPY --chown=1000:1000 --from=sdk /tmp /cache
# Ensure the ARG variables are used in the layer to prevent reuse by other builds.
COPY --chown=1000:1000 Twoliter.toml /cache/.${PACKAGE}.${ARCH}.${TOKEN}

# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=
# Generate the expected RPM macros and bconds.
FROM sdk as rpm-macros-and-bconds
ARG VARIANT
ARG VARIANT_PLATFORM
ARG VARIANT_RUNTIME
ARG VARIANT_FAMILY
ARG VARIANT_FLAVOR
ARG GRUB_SET_PRIVATE_VAR
ARG UEFI_SECURE_BOOT
ARG SYSTEMD_NETWORKD
ARG UNIFIED_CGROUP_HIERARCHY
ARG XFS_DATA_PARTITION
ARG FIPS

USER builder
WORKDIR /home/builder
RUN \
export RPM_MACROS="generated.rpmmacros" \
&& export RPM_BCONDS="generated.bconds" \
&& echo "%_cross_variant ${VARIANT}" > "${RPM_MACROS}" \
&& echo "%_cross_variant_platform ${VARIANT_PLATFORM}" >> "${RPM_MACROS}" \
&& echo "%_cross_variant_runtime ${VARIANT_RUNTIME}" >> "${RPM_MACROS}" \
&& echo "%_cross_variant_family ${VARIANT_FAMILY}" >> "${RPM_MACROS}" \
&& echo "%_cross_variant_flavor ${VARIANT_FLAVOR:-none}" >> "${RPM_MACROS}" \
&& echo "%_topdir /home/builder/rpmbuild" >> "${RPM_MACROS}" \
&& echo "%bcond_without $(V=${VARIANT_PLATFORM,,}; echo ${V//-/_})_platform" > "${RPM_BCONDS}" \
&& echo "%bcond_without $(V=${VARIANT_RUNTIME,,}; echo ${V//-/_})_runtime" >> "${RPM_BCONDS}" \
&& echo "%bcond_without $(V=${VARIANT_FAMILY,,}; echo ${V//-/_})_family" >> "${RPM_BCONDS}" \
&& echo "%bcond_without $(V=${VARIANT_FLAVOR:-no}; V=${V,,}; echo ${V//-/_})_flavor" >> "${RPM_BCONDS}" \
&& echo -e -n "${GRUB_SET_PRIVATE_VAR:+%bcond_without grub_set_private_var\n}" >> "${RPM_BCONDS}" \
&& echo -e -n "${FIPS:+%bcond_without fips\n}" >> "${RPM_BCONDS}" \
&& echo -e -n "${UEFI_SECURE_BOOT:+%bcond_without uefi_secure_boot\n}" >> "${RPM_BCONDS}" \
&& echo -e -n "${SYSTEMD_NETWORKD:+%bcond_without systemd_networkd\n}" >> "${RPM_BCONDS}" \
&& echo -e -n "${UNIFIED_CGROUP_HIERARCHY:+%bcond_without unified_cgroup_hierarchy\n}" >> "${RPM_BCONDS}" \
&& echo -e -n "${XFS_DATA_PARTITION:+%bcond_without xfs_data_partition\n}" >> "${RPM_BCONDS}"

# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=
# Builds an RPM package from a spec file.
FROM sdk AS rpmbuild
Expand All @@ -77,14 +40,10 @@ ARG PACKAGE_DEPENDENCIES
ARG KIT_DEPENDENCIES
ARG ARCH
ARG NOCACHE
ARG VARIANT
ARG SYSTEMD_NETWORKD
ARG BUILD_ID
ARG BUILD_ID_TIMESTAMP
ARG BUILD_EPOCH
ENV SYSTEMD_NETWORKD=${SYSTEMD_NETWORKD}
ENV VARIANT=${VARIANT}
ENV BUILD_ID=${BUILD_ID}
ENV BUILD_ID=${BUILD_ID}
ENV BUILD_ID_TIMESTAMP=${BUILD_ID_TIMESTAMP}
ENV BUILD_EPOCH=${BUILD_EPOCH}
WORKDIR /home/builder
Expand All @@ -93,18 +52,16 @@ USER builder
ENV PACKAGE=${PACKAGE} ARCH=${ARCH}
COPY ./packages/${PACKAGE}/ .

COPY --chown=builder --from=rpm-macros-and-bconds /home/builder/generated.* .

# Merge generated bconds with the package spec, and put sources in the right place. Set Epoch values for the package
# Copy over the target-specific macros, and put sources in the right place. Set Epoch values for the package
# and all of its subpackages. Additionally, for packages declaring explicit package versions requirements via
# "{Requires,Conflicts,Obsoletes}: = ...", ensure the epoch for the specified package is set.
# "[Requires|Conflicts|Obsoletes]:.*[=>,>,<,<=,=]\) \(\%{version}\-\%{release}$\)" matches any line in a form like:
# "Requires: %{name}-modules = %{version}-%{release}". The full `sed` expression below captures this match into
# two groups and insert the Epoch value such that the result is "Requires: %{name}-modules = EPOCH:%{version}-%{release}"
RUN \
cat "/usr/lib/rpm/platform/${ARCH}-bottlerocket/macros" generated.rpmmacros > .rpmmacros \
cp "/usr/lib/rpm/platform/${ARCH}-bottlerocket/macros" .rpmmacros \
&& cp ${PACKAGE}.spec rpmbuild/SPECS/${PACKAGE}.spec \
&& echo "Epoch: ${BUILD_EPOCH}" >> rpmbuild/SPECS/${PACKAGE}.spec \
&& cat generated.bconds ${PACKAGE}.spec >> rpmbuild/SPECS/${PACKAGE}.spec \
&& sed -i "/^%package\s.*$/a Epoch: ${BUILD_EPOCH}" rpmbuild/SPECS/${PACKAGE}.spec \
&& sed -i "s;\([Requires|Conflicts|Obsoletes]:.*[=>,>,<,<=,=]\) \(\%{version}\-\%{release}$\);\1 ${BUILD_EPOCH}:\2;" rpmbuild/SPECS/${PACKAGE}.spec \
&& find . -maxdepth 1 -not -path '*/\.*' -type f -exec mv {} rpmbuild/SOURCES/ \; \
Expand Down Expand Up @@ -143,7 +100,6 @@ RUN --mount=target=/host \
# Ensure that the target binutils that `find-debuginfo.sh` uses are present in $PATH.
ENV PATH="/usr/${ARCH}-bottlerocket-linux-gnu/debuginfo/bin:${PATH}"

# We use the "nocache" writable space to generate code where necessary.
USER builder
RUN --mount=source=.cargo,target=/home/builder/.cargo \
--mount=type=cache,target=/home/builder/.cache,from=cache,source=/cache \
Expand Down Expand Up @@ -203,9 +159,47 @@ COPY --from=kitbuild /home/builder/output/. /output/
# Section 3: The following build stages are used to create a Bottlerocket image once all of
# the rpm files have been created by repeatedly using Sections 1 and 2.

# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=
# Generate the expected RPM macros and bconds.
FROM sdk as rpm-macros-and-bconds
ARG VARIANT
ARG VARIANT_PLATFORM
ARG VARIANT_RUNTIME
ARG VARIANT_FAMILY
ARG VARIANT_FLAVOR
ARG GRUB_SET_PRIVATE_VAR
ARG UEFI_SECURE_BOOT
ARG SYSTEMD_NETWORKD
ARG UNIFIED_CGROUP_HIERARCHY
ARG XFS_DATA_PARTITION
ARG FIPS

USER builder
WORKDIR /home/builder
RUN \
export RPM_MACROS="generated.rpmmacros" \
&& export RPM_BCONDS="generated.bconds" \
&& echo "%_cross_variant ${VARIANT}" > "${RPM_MACROS}" \
&& echo "%_cross_variant_platform ${VARIANT_PLATFORM}" >> "${RPM_MACROS}" \
&& echo "%_cross_variant_runtime ${VARIANT_RUNTIME}" >> "${RPM_MACROS}" \
&& echo "%_cross_variant_family ${VARIANT_FAMILY}" >> "${RPM_MACROS}" \
&& echo "%_cross_variant_flavor ${VARIANT_FLAVOR:-none}" >> "${RPM_MACROS}" \
&& echo "%_topdir /home/builder/rpmbuild" >> "${RPM_MACROS}" \
&& echo "%bcond_without $(V=${VARIANT_PLATFORM,,}; echo ${V//-/_})_platform" > "${RPM_BCONDS}" \
&& echo "%bcond_without $(V=${VARIANT_RUNTIME,,}; echo ${V//-/_})_runtime" >> "${RPM_BCONDS}" \
&& echo "%bcond_without $(V=${VARIANT_FAMILY,,}; echo ${V//-/_})_family" >> "${RPM_BCONDS}" \
&& echo "%bcond_without $(V=${VARIANT_FLAVOR:-no}; V=${V,,}; echo ${V//-/_})_flavor" >> "${RPM_BCONDS}" \
&& echo -e -n "${GRUB_SET_PRIVATE_VAR:+%bcond_without grub_set_private_var\n}" >> "${RPM_BCONDS}" \
&& echo -e -n "${FIPS:+%bcond_without fips\n}" >> "${RPM_BCONDS}" \
&& echo -e -n "${UEFI_SECURE_BOOT:+%bcond_without uefi_secure_boot\n}" >> "${RPM_BCONDS}" \
&& echo -e -n "${SYSTEMD_NETWORKD:+%bcond_without systemd_networkd\n}" >> "${RPM_BCONDS}" \
&& echo -e -n "${UNIFIED_CGROUP_HIERARCHY:+%bcond_without unified_cgroup_hierarchy\n}" >> "${RPM_BCONDS}" \
&& echo -e -n "${XFS_DATA_PARTITION:+%bcond_without xfs_data_partition\n}" >> "${RPM_BCONDS}"


# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=
# Creates an RPM repository from packages created in Section 1 and kits from Section 2.
FROM sdk AS repobuild
FROM rpm-macros-and-bconds AS repobuild
# The list of packages from the variant Cargo.toml package.metadata.build-variant.packages section.
ARG PACKAGES
# The complete list of non-kit packages required by way of pure package-to-package dependencies.
Expand All @@ -217,8 +211,6 @@ ARG NOCACHE
WORKDIR /home/builder
USER builder

COPY --chown=builder --from=rpm-macros-and-bconds /home/builder/generated.* .

# Build the metadata RPM for the variant.
RUN --mount=target=/host \
cat "/usr/lib/rpm/platform/${ARCH}-bottlerocket/macros" generated.rpmmacros > .rpmmacros \
Expand Down

0 comments on commit 2721a63

Please sign in to comment.