From 9509471487cf800bdbee1607567e2a8bf65f0ed2 Mon Sep 17 00:00:00 2001 From: thesayyn Date: Mon, 25 Mar 2024 13:40:25 -0700 Subject: [PATCH 1/4] feat: use headers attribute if available --- .bazelrc | 2 - .bazelversion | 2 +- .github/workflows/ci.yaml | 96 ++++++++++++++++++++++++++----- .github/workflows/release_prep.sh | 23 ++++---- MODULE.bazel | 18 ++++-- apko/BUILD.bazel | 12 ++-- apko/private/BUILD.bazel | 6 ++ apko/private/apk.bzl | 40 ++++++++++--- apko/private/bazelversion.bzl | 45 +++++++++++++++ docs/initial-setup.md | 5 +- 10 files changed, 198 insertions(+), 51 deletions(-) create mode 100644 apko/private/bazelversion.bzl diff --git a/.bazelrc b/.bazelrc index cc9a804..729982b 100644 --- a/.bazelrc +++ b/.bazelrc @@ -7,8 +7,6 @@ common --enable_bzlmod # Disable lockfile for now. It is unstable. common --lockfile_mode=off -# Required for rules_apko to make range requests -try-import %workspace%/.apko/.bazelrc # Load any settings specific to the current user. # .bazelrc.user should appear in .gitignore so that settings are not shared with team members diff --git a/.bazelversion b/.bazelversion index fb58dfb..8590cbe 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1,4 +1,4 @@ -6.3.0 +7.1.0 # The first line of this file is used by Bazelisk and Bazel to be sure # the right version of Bazel is used to build and test this repo. # This also defines which version is used on CI. diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 66c6f24..763d0d9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,17 +16,87 @@ concurrency: cancel-in-progress: true jobs: + # matrix-prep-* steps generate JSON used to create a dynamic actions matrix. + # Inspired from + # https://stackoverflow.com/questions/65384420/how-to-make-a-github-action-matrix-element-conditional + + matrix-prep-os: + # Prepares the 'os' axis of the test matrix, to reduce costs since GitHub hosted runners cost more on some platforms. + # https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions#included-storage-and-minutes + runs-on: ubuntu-latest + steps: + - id: linux + run: echo "os=ubuntu-latest" >> $GITHUB_OUTPUT + - id: windows + run: echo "os=windows-latest" >> $GITHUB_OUTPUT + # Only run on main branch (or PR branches that contain 'windows') to minimize Windows minutes (billed at 2X) + if: (github.ref == 'refs/heads/main' || contains(github.head_ref, 'windows')) && !inputs.exclude_windows + - id: macos + run: echo "os=macos-latest" >> $GITHUB_OUTPUT + # Only run on main branch (or PR branches that contain 'macos') to minimize macOS minutes (billed at 10X) + if: github.ref == 'refs/heads/main' || contains(github.head_ref, 'macos') + outputs: + # Will look like ["ubuntu-latest", "windows-latest", "macos-latest"] + os: ${{ toJSON(steps.*.outputs.os) }} + + matrix-prep-bazelversion: + # Prepares the 'bazelversion' axis of the test matrix + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + # NB: we assume this is Bazel 7 + - id: bazel_from_bazelversion + run: echo "bazelversion=$(head -n 1 .bazelversion)" >> $GITHUB_OUTPUT + - id: bazel_6 + run: echo "bazelversion=6.4.0" >> $GITHUB_OUTPUT + outputs: + # Will look like ["", "x.y.z"] + bazelversions: ${{ toJSON(steps.*.outputs.bazelversion) }} + test: - uses: bazel-contrib/.github/.github/workflows/bazel.yaml@29e53247c6366e30acbedfc767f58f79fc05836c - with: - folders: | - [ - ".", - "e2e/smoke" - ] - exclude: | - [ - {"bazelversion": "5.4.0"}, - {"os": "windows-latest"}, - {"os": "macos-latest"} - ] + # The type of runner that the job will run on + runs-on: ${{ matrix.os }} + + needs: + - matrix-prep-bazelversion + - matrix-prep-os + + # Run bazel test in each workspace with each version of Bazel supported + strategy: + fail-fast: false + matrix: + os: ${{ fromJSON(needs.matrix-prep-os.outputs.os) }} + bazelversion: ${{ fromJSON(needs.matrix-prep-bazelversion.outputs.bazelversions) }} + folder: + - . + - e2e/smoke + bzlmodEnabled: [true, false] + exclude: + # Root module is BZLMOD only, do not test it without bzlmod enabled. + - bzlmodEnable: false + folder: . + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v4 + + - uses: bazel-contrib/setup-bazel@0.8.0 + with: + repository-cache: true + bazelrc: | + common --announce_rc --color=yes --enable_bzlmod=${{ matrix.bzlmodEnabled }} + ${{ (matrix.bazelversion == '6.4.0' && 'try-import %workspace%/.apko/.bazelrc') || '' }} + + - name: Configure Bazel version + working-directory: ${{ matrix.folder }} + run: echo "${{ matrix.bazelversion }}" > .bazelversion + + # See https://github.com/bazel-contrib/publish-to-bcr#including-patches + - name: verify bcr patches + if: matrix.bzlmodEnabled && hashFiles('.bcr/patches/*.patch') != '' + run: patch --dry-run -p1 < .bcr/patches/*.patch + + - name: Test + working-directory: ${{ matrix.folder }} + run: bazel test //... diff --git a/.github/workflows/release_prep.sh b/.github/workflows/release_prep.sh index 265b7c1..9b492c9 100755 --- a/.github/workflows/release_prep.sh +++ b/.github/workflows/release_prep.sh @@ -12,7 +12,17 @@ git archive --format=tar --prefix=${PREFIX}/ ${TAG} | gzip > $ARCHIVE SHA=$(shasum -a 256 $ARCHIVE | awk '{print $1}') cat << EOF -## Using Bzlmod with Bazel 6 + +## Initial setup (when using with Bazel < 7.1) + +rules_apko requires a one-time setup to configure bazel to be able to make partial fetches. + +Follow https://github.com/chainguard-dev/rules_apko/blob/main/docs/initial-setup.md for the setup. + +EOF + +cat << EOF +## Using Bzlmod 1. Enable with \`common --enable_bzlmod\` in \`.bazelrc\`. 2. Add to your \`MODULE.bazel\` file: @@ -37,14 +47,3 @@ EOF awk 'f;/--SNIP--/{f=1}' e2e/smoke/WORKSPACE.bazel echo "\`\`\`" - - -cat << EOF - -## Initial setup - -rules_apko requires a one-time setup to configure bazel to be able to make partial fetches. - -Follow https://github.com/chainguard-dev/rules_apko/blob/main/docs/initial-setup.md for the setup. - -EOF \ No newline at end of file diff --git a/MODULE.bazel b/MODULE.bazel index 4ad6b33..970783b 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -6,16 +6,22 @@ module( compatibility_level = 1, ) -bazel_dep(name = "bazel_skylib", version = "1.4.2") -bazel_dep(name = "platforms", version = "0.0.5") -bazel_dep(name = "aspect_bazel_lib", version = "1.34.5") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "aspect_bazel_lib", version = "1.40.3") + +# We need this fix https://github.com/aspect-build/bazel-lib/pull/713 to be able to generate docs +single_version_override( + module_name = "aspect_bazel_lib", + version = "2.5.3", +) bazel_dep(name = "container_structure_test", version = "1.15.0", dev_dependency = True) bazel_dep(name = "rules_pkg", version = "0.7.0", dev_dependency = True) bazel_dep(name = "rules_oci", version = "1.3.3", dev_dependency = True) -bazel_dep(name = "gazelle", version = "0.29.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.4.1", dev_dependency = True) -bazel_dep(name = "buildifier_prebuilt", version = "6.1.0", dev_dependency = True) +bazel_dep(name = "gazelle", version = "0.34.0", dev_dependency = True, repo_name = "bazel_gazelle") +bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.5.0", dev_dependency = True) +bazel_dep(name = "buildifier_prebuilt", version = "6.1.2", dev_dependency = True) toolchain = use_extension("//apko:extensions.bzl", "apko") toolchain.toolchain(apko_version = "v0.14.1") diff --git a/apko/BUILD.bazel b/apko/BUILD.bazel index 8e36bbe..3c3debb 100644 --- a/apko/BUILD.bazel +++ b/apko/BUILD.bazel @@ -63,12 +63,6 @@ bzl_library( ], ) -bzl_library( - name = "toolchain", - srcs = ["toolchain.bzl"], - visibility = ["//visibility:public"], -) - # Enables calling apko tool directly by bazel. # To resolve given `./apko.yaml` file into `./apko.lock.json`, once can call: # e.g. (cd ./examples/oci; bazel run @rules_apko//apko lock ./apko.yaml) @@ -76,3 +70,9 @@ apko_run( name = "apko", visibility = ["//visibility:public"], ) + +bzl_library( + name = "toolchain", + srcs = ["toolchain.bzl"], + visibility = ["//visibility:public"], +) diff --git a/apko/private/BUILD.bazel b/apko/private/BUILD.bazel index df424ab..4f113f9 100644 --- a/apko/private/BUILD.bazel +++ b/apko/private/BUILD.bazel @@ -67,3 +67,9 @@ bzl_library( srcs = ["versions.bzl"], visibility = ["//apko:__subpackages__"], ) + +bzl_library( + name = "bazelversion", + srcs = ["bazelversion.bzl"], + visibility = ["//apko:__subpackages__"], +) diff --git a/apko/private/apk.bzl b/apko/private/apk.bzl index ebd9780..59b040f 100644 --- a/apko/private/apk.bzl +++ b/apko/private/apk.bzl @@ -1,5 +1,6 @@ "Repository rules for importing remote apk packages" +load(":bazelversion.bzl", "bazel_version_gte") load(":util.bzl", "util") APK_IMPORT_TMPL = """\ @@ -16,8 +17,10 @@ def _range(url, range): def _check_initial_setup(rctx): output = rctx.path(".rangecheck/output") - rctx.download( - url = [_range(rctx.attr.url, "bytes=0-0")], + _download( + rctx, + url = rctx.attr.url, + rng = "bytes=0-0", output = output, ) r = rctx.execute(["wc", "-c", output]) @@ -39,6 +42,19 @@ To resolve this issue and enable partial package fetching, please follow the ste """.format(bytes[0])) +def _download(rctx, url, rng, **kwargs): + if bazel_version_gte("7.1.0"): + return rctx.download( + url = [url], + headers = {"Range": [rng]}, + **kwargs + ) + else: + return rctx.download( + url = [_range(url, rng)], + **kwargs + ) + def _apk_import_impl(rctx): repo = util.repo_url(rctx.attr.url, rctx.attr.architecture) repo_escaped = util.url_escape(repo) @@ -53,19 +69,25 @@ def _apk_import_impl(rctx): data_output = "{}/{}.dat.tar.gz".format(output, data_sha256) apk_output = "{}/{}/{}-{}.apk".format(repo_escaped, rctx.attr.architecture, rctx.attr.package_name, rctx.attr.version) - rctx.download( - url = [_range(rctx.attr.url, rctx.attr.signature_range)], + _download( + rctx, + url = rctx.attr.url, + rng = rctx.attr.signature_range, output = sig_output, # TODO: signatures does not have stable checksums. find a way to fail gracefully. - integrity = rctx.attr.signature_checksum, + # integrity = rctx.attr.signature_checksum, ) - rctx.download( - url = [_range(rctx.attr.url, rctx.attr.control_range)], + _download( + rctx, + url = rctx.attr.url, + rng = rctx.attr.control_range, output = control_output, integrity = rctx.attr.control_checksum, ) - rctx.download( - url = [_range(rctx.attr.url, rctx.attr.data_range)], + _download( + rctx, + url = rctx.attr.url, + rng = rctx.attr.data_range, output = data_output, integrity = rctx.attr.data_checksum, ) diff --git a/apko/private/bazelversion.bzl b/apko/private/bazelversion.bzl new file mode 100644 index 0000000..bb0fbed --- /dev/null +++ b/apko/private/bazelversion.bzl @@ -0,0 +1,45 @@ +"""Internal functions to parse versions.""" + +# Taken from https://github.com/bazel-contrib/bazel_features/blob/main/private/parse.bzl +# We can't use bazel_features because it requires two loads and macro calls in the WORKSPACE +# file but rules_apko users make one load and two macro calls where marcros exported from +# the same file which makes it not possible to add bazel_features to `repositories.bzl` file +# and load from it in the repositories.bzl file. +# +# TODO(2.0): depend on bazel_features directly by splitting the repositories.bzl file into two. +# https://github.com/chainguard-dev/rules_apko/issues/55 + +def _safe_int(s, v): + if not s.isdigit(): + fail("invalid Bazel version '{}': non-numeric segment '{}'".format(v, s)) + return int(s) + +def _partition(s): + for i in range(len(s)): + c = s[i] + if c == "-": + return s[:i], s[i + 1:] + if not c.isdigit() and c != ".": + return s[:i], s[i:] + return s, "" + +def parse_version(v): + """Parses the given Bazel version string into a comparable value. + + Args: + v (str): version string + + Returns: + a triple ([major, minor, patch], is_released, prerelease) + """ + if not v: + # An empty string is treated as a "dev version", which is greater than anything. + v = "999999.999999.999999" + release, prerelease = _partition(v.partition(" ")[0]) + segments = release.split(".") + if len(segments) != 3: + fail("invalid Bazel version '{}': got {} dot-separated segments, want 3".format(v, len(segments))) + return [_safe_int(s, v) for s in segments], not prerelease, prerelease + +def bazel_version_gte(version): + return parse_version(native.bazel_version) >= parse_version(version) diff --git a/docs/initial-setup.md b/docs/initial-setup.md index b01b100..fbec31e 100644 --- a/docs/initial-setup.md +++ b/docs/initial-setup.md @@ -1,8 +1,9 @@ # Initial setup -rules_apko requires a one-time setup to configure bazel to be able to make partial fetches. +You can skip initial setup if you are using Bazel 7.1 or above. Users who are still Bazel 6.x should perform this one time initial setup. +If you have already performed this initial setup but have already upgraded to Bazel >=7.1, you can revert changes proposed by this document. -> See the issue: https://github.com/bazelbuild/bazel/issues/17829 +rules_apko requires a one-time setup to configure bazel to be able to make partial fetches. Paste this into your root BUILD file From 49239b7583d75ab1829fb40a208e91b83c1d3718 Mon Sep 17 00:00:00 2001 From: thesayyn Date: Mon, 25 Mar 2024 14:31:34 -0700 Subject: [PATCH 2/4] use syklib versions --- .github/workflows/ci.yaml | 6 ++--- apko/private/BUILD.bazel | 6 ----- apko/private/apk.bzl | 4 ++-- apko/private/bazelversion.bzl | 45 ----------------------------------- 4 files changed, 5 insertions(+), 56 deletions(-) delete mode 100644 apko/private/bazelversion.bzl diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 763d0d9..ba22420 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,4 +1,4 @@ -name: CI +name: test # Controls when the action will run. on: @@ -48,7 +48,7 @@ jobs: - id: bazel_from_bazelversion run: echo "bazelversion=$(head -n 1 .bazelversion)" >> $GITHUB_OUTPUT - id: bazel_6 - run: echo "bazelversion=6.4.0" >> $GITHUB_OUTPUT + run: echo "bazelversion=6.3.0" >> $GITHUB_OUTPUT outputs: # Will look like ["", "x.y.z"] bazelversions: ${{ toJSON(steps.*.outputs.bazelversion) }} @@ -73,7 +73,7 @@ jobs: bzlmodEnabled: [true, false] exclude: # Root module is BZLMOD only, do not test it without bzlmod enabled. - - bzlmodEnable: false + - bzlmodEnabled: false folder: . # Steps represent a sequence of tasks that will be executed as part of the job diff --git a/apko/private/BUILD.bazel b/apko/private/BUILD.bazel index 4f113f9..df424ab 100644 --- a/apko/private/BUILD.bazel +++ b/apko/private/BUILD.bazel @@ -67,9 +67,3 @@ bzl_library( srcs = ["versions.bzl"], visibility = ["//apko:__subpackages__"], ) - -bzl_library( - name = "bazelversion", - srcs = ["bazelversion.bzl"], - visibility = ["//apko:__subpackages__"], -) diff --git a/apko/private/apk.bzl b/apko/private/apk.bzl index 59b040f..a3d927a 100644 --- a/apko/private/apk.bzl +++ b/apko/private/apk.bzl @@ -1,6 +1,6 @@ "Repository rules for importing remote apk packages" -load(":bazelversion.bzl", "bazel_version_gte") +load("@bazel_skylib//lib:versions.bzl", "versions") load(":util.bzl", "util") APK_IMPORT_TMPL = """\ @@ -43,7 +43,7 @@ To resolve this issue and enable partial package fetching, please follow the ste """.format(bytes[0])) def _download(rctx, url, rng, **kwargs): - if bazel_version_gte("7.1.0"): + if versions.is_at_least("7.1.0", native.bazel_version): return rctx.download( url = [url], headers = {"Range": [rng]}, diff --git a/apko/private/bazelversion.bzl b/apko/private/bazelversion.bzl deleted file mode 100644 index bb0fbed..0000000 --- a/apko/private/bazelversion.bzl +++ /dev/null @@ -1,45 +0,0 @@ -"""Internal functions to parse versions.""" - -# Taken from https://github.com/bazel-contrib/bazel_features/blob/main/private/parse.bzl -# We can't use bazel_features because it requires two loads and macro calls in the WORKSPACE -# file but rules_apko users make one load and two macro calls where marcros exported from -# the same file which makes it not possible to add bazel_features to `repositories.bzl` file -# and load from it in the repositories.bzl file. -# -# TODO(2.0): depend on bazel_features directly by splitting the repositories.bzl file into two. -# https://github.com/chainguard-dev/rules_apko/issues/55 - -def _safe_int(s, v): - if not s.isdigit(): - fail("invalid Bazel version '{}': non-numeric segment '{}'".format(v, s)) - return int(s) - -def _partition(s): - for i in range(len(s)): - c = s[i] - if c == "-": - return s[:i], s[i + 1:] - if not c.isdigit() and c != ".": - return s[:i], s[i:] - return s, "" - -def parse_version(v): - """Parses the given Bazel version string into a comparable value. - - Args: - v (str): version string - - Returns: - a triple ([major, minor, patch], is_released, prerelease) - """ - if not v: - # An empty string is treated as a "dev version", which is greater than anything. - v = "999999.999999.999999" - release, prerelease = _partition(v.partition(" ")[0]) - segments = release.split(".") - if len(segments) != 3: - fail("invalid Bazel version '{}': got {} dot-separated segments, want 3".format(v, len(segments))) - return [_safe_int(s, v) for s in segments], not prerelease, prerelease - -def bazel_version_gte(version): - return parse_version(native.bazel_version) >= parse_version(version) From 9ef6d3aa4bf52a4566a9d1731332c9354acfcbd5 Mon Sep 17 00:00:00 2001 From: thesayyn Date: Thu, 16 May 2024 08:51:00 -0700 Subject: [PATCH 3/4] add ci step --- .github/workflows/ci.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ba22420..87f4607 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -97,6 +97,10 @@ jobs: if: matrix.bzlmodEnabled && hashFiles('.bcr/patches/*.patch') != '' run: patch --dry-run -p1 < .bcr/patches/*.patch + # Required for rules_apko to make range requests + - name: Add bazel 6 workaround + run: echo 'try-import %workspace%/.apko/.bazelrc' >> .bazelrc + - name: Test working-directory: ${{ matrix.folder }} run: bazel test //... From 3d0e6d4b9062bb1d9e32795045b3c3b03ab95bff Mon Sep 17 00:00:00 2001 From: thesayyn Date: Thu, 16 May 2024 09:10:03 -0700 Subject: [PATCH 4/4] add ci step --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 87f4607..e25e556 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -99,6 +99,7 @@ jobs: # Required for rules_apko to make range requests - name: Add bazel 6 workaround + if: ${{ matrix.bazelversion == '6.3.0' }} run: echo 'try-import %workspace%/.apko/.bazelrc' >> .bazelrc - name: Test