From dc33808932ec9ed2dd1a8d29fccf53328ab5b59e Mon Sep 17 00:00:00 2001 From: Julio Merino Date: Thu, 9 Feb 2023 16:10:44 -0800 Subject: [PATCH] Implement support for BAZELISK_FORMAT_URL This new configuration setting provides a format-like string to compute the URL from which to fetch Bazel. Takes precedence over BAZELISK_BASE_URL as this is a more general concept. Fixes #423. --- README.md | 11 ++++++- bazelisk_test.sh | 23 ++++++++++++-- core/BUILD | 5 +++- core/core.go | 54 ++++++++++++++++++--------------- core/repositories.go | 60 +++++++++++++++++++++++++++++++++++++ core/repositories_test.go | 63 +++++++++++++++++++++++++++++++++++++++ httputil/BUILD | 6 ++-- platforms/BUILD | 6 ++++ platforms/platforms.go | 27 +++++++++++++---- 9 files changed, 217 insertions(+), 38 deletions(-) create mode 100644 core/repositories_test.go diff --git a/README.md b/README.md index 93f63708..c8337d31 100644 --- a/README.md +++ b/README.md @@ -77,11 +77,20 @@ By default Bazelisk retrieves Bazel releases, release candidates and binaries bu As mentioned in the previous section, the `/` version format allows you to use your own Bazel fork hosted on GitHub: -If you want to create a fork with your own releases, you have to follow the naming conventions that we use in `bazelbuild/bazel` for the binary file names. +If you want to create a fork with your own releases, you should follow the naming conventions that we use in `bazelbuild/bazel` for the binary file names as this results in predictable URLs that are similar to the official ones. The URL format looks like `https://github.com//bazel/releases/download//`. You can also override the URL by setting the environment variable `$BAZELISK_BASE_URL`. Bazelisk will then append `//` to the base URL instead of using the official release server. Bazelisk will read file [`~/.netrc`](https://everything.curl.dev/usingcurl/netrc) for credentials for Basic authentication. +If for any reason none of this works, you can also override the URL format altogether by setting the environment variable `$BAZELISK_FORMAT_URL`. This variable takes a format-like string with placeholders and performs the following replacements to compute the download URL: + +- `%e`: Extension suffix, such as the empty string or `.exe`. +- `%m`: Machine architecture name, such as `arm64` or `x86_64`. +- `%o`: Operating system name, such as `darwin` or `linux`. +- `%v`: Bazel version as determined by Bazelisk. +- `%%`: Literal `%` for escaping purposes. +- All other characters after `%` are reserved for future use and result in a processing error. + ## Ensuring that your developers use Bazelisk rather than Bazel Bazel installers typically provide Bazel's [shell wrapper script] as the `bazel` on the PATH. diff --git a/bazelisk_test.sh b/bazelisk_test.sh index ad99dbe1..4e1d4f9e 100755 --- a/bazelisk_test.sh +++ b/bazelisk_test.sh @@ -197,7 +197,20 @@ function test_bazel_version_from_file() { (echo "FAIL: Expected to find 'Build label: 5.0.0' in the output of 'bazelisk version'"; exit 1) } -function test_bazel_version_from_url() { +function test_bazel_version_from_format_url() { + setup + + echo "0.19.0" > .bazelversion + + BAZELISK_FORMAT_URL="https://github.com/bazelbuild/bazel/releases/download/%v/bazel-%v-%o-%m%e" \ + BAZELISK_HOME="$BAZELISK_HOME" \ + bazelisk version 2>&1 | tee log + + grep "Build label: 0.19.0" log || \ + (echo "FAIL: Expected to find 'Build label: 0.19.0' in the output of 'bazelisk version'"; exit 1) +} + +function test_bazel_version_from_base_url() { setup echo "0.19.0" > .bazelversion @@ -383,8 +396,12 @@ if [[ $BAZELISK_VERSION == "GO" ]]; then test_bazel_last_rc echo - echo "# test_bazel_version_from_url" - test_bazel_version_from_url + echo "# test_bazel_version_from_format_url" + test_bazel_version_from_format_url + echo + + echo "# test_bazel_version_from_base_url" + test_bazel_version_from_base_url echo echo "# test_bazel_version_prefer_environment_to_bazeliskrc" diff --git a/core/BUILD b/core/BUILD index 2f5ec601..b941e3d2 100644 --- a/core/BUILD +++ b/core/BUILD @@ -19,6 +19,9 @@ go_library( go_test( name = "go_default_test", - srcs = ["core_test.go"], + srcs = [ + "core_test.go", + "repositories_test.go", + ], embed = [":go_default_library"], ) diff --git a/core/core.go b/core/core.go index 01f5f685..df5f6d9d 100644 --- a/core/core.go +++ b/core/core.go @@ -413,8 +413,14 @@ func downloadBazel(version string, baseDirectory string, repos *Repositories, do destFile := "bazel" + platforms.DetermineExecutableFilenameSuffix() destinationDir := filepath.Join(baseDirectory, pathSegment, "bin") - if url := GetEnvOrConfig(BaseURLEnv); url != "" { - return repos.DownloadFromBaseURL(url, version, destinationDir, destFile) + baseURL := GetEnvOrConfig(BaseURLEnv) + formatURL := GetEnvOrConfig(FormatURLEnv) + if baseURL != "" && formatURL != "" { + return "", fmt.Errorf("cannot set %s and %s at once", BaseURLEnv, FormatURLEnv) + } else if formatURL != "" { + return repos.DownloadFromFormatURL(formatURL, version, destinationDir, destFile) + } else if baseURL != "" { + return repos.DownloadFromBaseURL(baseURL, version, destinationDir, destFile) } return downloader(destinationDir, destFile) @@ -471,12 +477,12 @@ func maybeDelegateToWrapperFromDir(bazel string, wd string, ignoreEnv bool) stri } if runtime.GOOS == "windows" { - powershellWrapper := filepath.Join(root, wrapperPath + ".ps1") + powershellWrapper := filepath.Join(root, wrapperPath+".ps1") if stat, err := os.Stat(powershellWrapper); err == nil && !stat.Mode().IsDir() { return powershellWrapper } - batchWrapper := filepath.Join(root, wrapperPath + ".bat") + batchWrapper := filepath.Join(root, wrapperPath+".bat") if stat, err := os.Stat(batchWrapper); err == nil && !stat.Mode().IsDir() { return batchWrapper } @@ -605,27 +611,27 @@ func insertArgs(baseArgs []string, newArgs []string) []string { func parseStartupOptions(baseArgs []string) []string { var result []string var BAZEL_COMMANDS = map[string]bool{ - "analyze-profile": true, - "aquery": true, - "build": true, + "analyze-profile": true, + "aquery": true, + "build": true, "canonicalize-flags": true, - "clean": true, - "coverage": true, - "cquery": true, - "dump": true, - "fetch": true, - "help": true, - "info": true, - "license": true, - "mobile-install": true, - "mod": true, - "print_action": true, - "query": true, - "run": true, - "shutdown": true, - "sync": true, - "test": true, - "version": true, + "clean": true, + "coverage": true, + "cquery": true, + "dump": true, + "fetch": true, + "help": true, + "info": true, + "license": true, + "mobile-install": true, + "mod": true, + "print_action": true, + "query": true, + "run": true, + "shutdown": true, + "sync": true, + "test": true, + "version": true, } // Arguments before a Bazel command are startup options. for _, arg := range baseArgs { diff --git a/core/repositories.go b/core/repositories.go index f0ab7327..9e83250e 100644 --- a/core/repositories.go +++ b/core/repositories.go @@ -13,6 +13,9 @@ import ( const ( // BaseURLEnv is the name of the environment variable that stores the base URL for downloads. BaseURLEnv = "BAZELISK_BASE_URL" + + // FormatURLEnv is the name of the environment variable that stores the format string to generate URLs for downloads. + FormatURLEnv = "BAZELISK_FORMAT_URL" ) // DownloadFunc downloads a specific Bazel binary to the given location and returns the absolute path. @@ -232,6 +235,63 @@ func (r *Repositories) DownloadFromBaseURL(baseURL, version, destDir, destFile s return httputil.DownloadBinary(url, destDir, destFile) } +func BuildURLFromFormat(formatURL, version string) (string, error) { + osName, err := platforms.DetermineOperatingSystem() + if err != nil { + return "", err + } + + machineName, err := platforms.DetermineArchitecture(osName, version) + if err != nil { + return "", err + } + + var b strings.Builder + b.Grow(len(formatURL) * 2) // Approximation. + for i := 0; i < len(formatURL); i++ { + ch := formatURL[i] + if ch == '%' { + i++ + if i == len(formatURL) { + return "", errors.New("trailing %") + } + + ch = formatURL[i] + switch ch { + case 'e': + b.WriteString(platforms.DetermineExecutableFilenameSuffix()) + case 'm': + b.WriteString(machineName) + case 'o': + b.WriteString(osName) + case 'v': + b.WriteString(version) + case '%': + b.WriteByte('%') + default: + return "", fmt.Errorf("unknown placeholder %%%c", ch) + } + } else { + b.WriteByte(ch) + } + } + return b.String(), nil +} + +// DownloadFromFormatURL can download Bazel binaries from a specific URL while ignoring the predefined repositories. +func (r *Repositories) DownloadFromFormatURL(formatURL, version, destDir, destFile string) (string, error) { + if formatURL == "" { + return "", fmt.Errorf("%s is not set", FormatURLEnv) + } + + url, err := BuildURLFromFormat(formatURL, version) + if err != nil { + return "", err + } + + return httputil.DownloadBinary(url, destDir, destFile) +} + // CreateRepositories creates a new Repositories instance with the given repositories. Any nil repository will be replaced by a dummy repository that raises an error whenever a download is attempted. func CreateRepositories(releases ReleaseRepo, candidates CandidateRepo, fork ForkRepo, commits CommitRepo, rolling RollingRepo, supportsBaseURL bool) *Repositories { repos := &Repositories{supportsBaseURL: supportsBaseURL} diff --git a/core/repositories_test.go b/core/repositories_test.go new file mode 100644 index 00000000..e35a0f1b --- /dev/null +++ b/core/repositories_test.go @@ -0,0 +1,63 @@ +package core + +import ( + "errors" + "fmt" + "testing" + + "github.com/bazelbuild/bazelisk/platforms" +) + +func TestBuildURLFromFormat(t *testing.T) { + osName, err := platforms.DetermineOperatingSystem() + if err != nil { + t.Fatalf("Cannot get operating system name: %v", err) + } + + version := "6.0.0" + + machineName, err := platforms.DetermineArchitecture(osName, version) + if err != nil { + t.Fatalf("Cannot get machine architecture name: %v", err) + } + + suffix := platforms.DetermineExecutableFilenameSuffix() + + type test struct { + format string + want string + wantErr error + } + + tests := []test{ + {format: "", want: ""}, + {format: "no/placeholders", want: "no/placeholders"}, + + {format: "%", wantErr: errors.New("trailing %")}, + {format: "%%", want: "%"}, + {format: "%%%%", want: "%%"}, + {format: "invalid/trailing/%", wantErr: errors.New("trailing %")}, + {format: "escaped%%placeholder", want: "escaped%placeholder"}, + + {format: "foo-%e-bar", want: fmt.Sprintf("foo-%s-bar", suffix)}, + {format: "foo-%m-bar", want: fmt.Sprintf("foo-%s-bar", machineName)}, + {format: "foo-%o-bar", want: fmt.Sprintf("foo-%s-bar", osName)}, + {format: "foo-%v-bar", want: fmt.Sprintf("foo-%s-bar", version)}, + + {format: "repeated %v %m %v", want: fmt.Sprintf("repeated %s %s %s", version, machineName, version)}, + + {format: "https://real.example.com/%e/%m/%o/%v#%%20trailing", want: fmt.Sprintf("https://real.example.com/%s/%s/%s/%s#%%20trailing", suffix, machineName, osName, version)}, + } + + for _, tc := range tests { + got, err := BuildURLFromFormat(tc.format, version) + if fmt.Sprintf("%v", err) != fmt.Sprintf("%v", tc.wantErr) { + if got != "" { + t.Errorf("format '%s': got non-empty '%s' on error", tc.format, got) + } + t.Errorf("format '%s': got error %v, want error %v", tc.format, err, tc.wantErr) + } else if got != tc.want { + t.Errorf("format '%s': got %s, want %s", tc.format, got, tc.want) + } + } +} diff --git a/httputil/BUILD b/httputil/BUILD index f168cffa..d48f2902 100644 --- a/httputil/BUILD +++ b/httputil/BUILD @@ -10,12 +10,12 @@ go_library( "fake.go", "httputil.go", ], + importpath = "github.com/bazelbuild/bazelisk/httputil", + visibility = ["//visibility:public"], deps = [ + "@com_github_bgentry_go_netrc//:go_default_library", "@com_github_mitchellh_go_homedir//:go_default_library", - "@com_github_bgentry_go_netrc//:go_default_library" ], - importpath = "github.com/bazelbuild/bazelisk/httputil", - visibility = ["//visibility:public"], ) go_test( diff --git a/platforms/BUILD b/platforms/BUILD index dcb655f5..8ab333f9 100644 --- a/platforms/BUILD +++ b/platforms/BUILD @@ -16,3 +16,9 @@ go_test( srcs = ["platforms_test.go"], embed = [":go_default_library"], ) + +go_test( + name = "go_default_test", + srcs = ["platforms_test.go"], + embed = [":go_default_library"], +) diff --git a/platforms/platforms.go b/platforms/platforms.go index 3c0fcbe0..2e918dcd 100644 --- a/platforms/platforms.go +++ b/platforms/platforms.go @@ -54,8 +54,7 @@ func DetermineExecutableFilenameSuffix() string { return filenameSuffix } -// DetermineBazelFilename returns the correct file name of a local Bazel binary. -func DetermineBazelFilename(version string, includeSuffix bool) (string, error) { +func DetermineArchitecture(osName, version string) (string, error) { var machineName string switch runtime.GOARCH { case "amd64": @@ -66,16 +65,32 @@ func DetermineBazelFilename(version string, includeSuffix bool) (string, error) return "", fmt.Errorf("unsupported machine architecture \"%s\", must be arm64 or x86_64", runtime.GOARCH) } - var osName string + if osName == "darwin" { + machineName = DarwinFallback(machineName, version) + } + + return machineName, nil +} + +func DetermineOperatingSystem() (string, error) { switch runtime.GOOS { case "darwin", "linux", "windows": - osName = runtime.GOOS + return runtime.GOOS, nil default: return "", fmt.Errorf("unsupported operating system \"%s\", must be Linux, macOS or Windows", runtime.GOOS) } +} - if osName == "darwin" { - machineName = DarwinFallback(machineName, version) +// DetermineBazelFilename returns the correct file name of a local Bazel binary. +func DetermineBazelFilename(version string, includeSuffix bool) (string, error) { + osName, err := DetermineOperatingSystem() + if err != nil { + return "", err + } + + machineName, err := DetermineArchitecture(osName, version) + if err != nil { + return "", err } var filenameSuffix string