diff --git a/examples/bzlmod/.bazelrc b/examples/bzlmod/.bazelrc index b8c233f98c..6f557e67b9 100644 --- a/examples/bzlmod/.bazelrc +++ b/examples/bzlmod/.bazelrc @@ -1,3 +1,10 @@ common --experimental_enable_bzlmod coverage --java_runtime_version=remotejdk_11 + +test --test_output=errors --enable_runfiles + +# Windows requires these for multi-python support: +build --enable_runfiles + +startup --windows_enable_symlinks diff --git a/examples/bzlmod/BUILD.bazel b/examples/bzlmod/BUILD.bazel index e1f5790631..86498226f9 100644 --- a/examples/bzlmod/BUILD.bazel +++ b/examples/bzlmod/BUILD.bazel @@ -1,9 +1,21 @@ +# Load various rules so that we can have bazel download +# various rulesets and dependencies. +# The `load` statement imports the symbol for the rule, in the defined +# ruleset. When the symbol is loaded you can use the rule. + +# The names @pip and @python_39 are values that are repository +# names. Those names are defined in the MODULES.bazel file. load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@pip//:requirements.bzl", "all_requirements", "all_whl_requirements", "requirement") -load("@python3_9//:defs.bzl", py_test_with_transition = "py_test") +load("@python_39//:defs.bzl", py_test_with_transition = "py_test") + +# This is not working yet till the toolchain hub registration is working +# load("@python_310//:defs.bzl", py_binary_310 = "py_binary") load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") +# This stanza calls a rule that generates targets for managing pip dependencies +# with pip-compile. compile_pip_requirements( name = "requirements", extra_args = ["--allow-unsafe"], @@ -12,6 +24,11 @@ compile_pip_requirements( requirements_windows = "requirements_windows.txt", ) +# The rules below are language specific rules defined in +# rules_python. See +# https://bazel.build/reference/be/python + +# see https://bazel.build/reference/be/python#py_library py_library( name = "lib", srcs = ["lib.py"], @@ -22,6 +39,7 @@ py_library( ], ) +# see https://bazel.build/reference/be/python#py_binary py_binary( name = "bzlmod", srcs = ["__main__.py"], @@ -32,6 +50,23 @@ py_binary( ], ) +# This is still WIP. Not working till we have the toolchain +# registration functioning. + +# This is used for testing mulitple versions of Python. This is +# used only when you need to support multiple versions of Python +# in the same project. +# py_binary_310( +# name = "main_310", +# srcs = ["__main__.py"], +# main = "__main__.py", +# visibility = ["//:__subpackages__"], +# deps = [ +# ":lib", +# ], +# ) + +# see https://bazel.build/reference/be/python#py_test py_test( name = "test", srcs = ["test.py"], @@ -46,6 +81,11 @@ py_test_with_transition( deps = [":lib"], ) +# This example is also used for integration tests within +# rules_python. We are using +# https://github.com/bazelbuild/bazel-skylib +# to run some of the tests. +# See: https://github.com/bazelbuild/bazel-skylib/blob/main/docs/build_test_doc.md build_test( name = "all_wheels", targets = all_whl_requirements, diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 145cebd276..bb4183bde2 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -11,32 +11,89 @@ local_path_override( path = "../..", ) +# This name is passed into python.toolchain and it's use_repo statement. +# We also use the same value in the python.host_python_interpreter call. +PYTHON_NAME_39 = "python_39" + +PYTHON_39_TOOLCHAINS = PYTHON_NAME_39 + "_toolchains" + +INTERPRETER_NAME_39 = "interpreter_39" + +PYTHON_NAME_310 = "python_310" + +PYTHON_310_TOOLCHAINS = PYTHON_NAME_310 + "_toolchains" + +INTERPRETER_NAME_310 = "interpreter_310" + +# We next initialize the python toolchain using the extension. +# You can set different Python versions in this block. python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( - name = "python3_9", + # This name is used in the various use_repo statements + # below, and in the local extension that is in this + # example. + name = PYTHON_NAME_39, configure_coverage_tool = True, + # Only set when you have mulitple toolchain versions. + is_default = True, python_version = "3.9", ) -use_repo(python, "python3_9") -use_repo(python, "python3_9_toolchains") +# We are also using a second version of Python in this project. +# Typically you will only need a single version of Python, but +# If you need a different vesion we support more than one. +# Note: we do not supporting using multiple pip extensions, this is +# work in progress. +python.toolchain( + name = PYTHON_NAME_310, + configure_coverage_tool = True, + python_version = "3.10", +) + +# use_repo imports one or more repos generated by the given module extension +# into the scope of the current module. We are importing the various repos +# created by the above python.toolchain calls. +use_repo( + python, + PYTHON_NAME_39, + PYTHON_39_TOOLCHAINS, + PYTHON_NAME_310, + PYTHON_310_TOOLCHAINS, +) + +# This call registers the Python toolchains. +# Note: there is work under way to move this code to within +# rules_python, and the user won't have to make this call, +# unless they are registering custom toolchains. register_toolchains( - "@python3_9_toolchains//:all", + "@{}//:all".format(PYTHON_39_TOOLCHAINS), + "@{}//:all".format(PYTHON_310_TOOLCHAINS), ) +# The interpreter extension discovers the platform specific Python binary. +# It creates a symlink to the binary, and we pass the label to the following +# pip.parse call. interpreter = use_extension("@rules_python//python/extensions:interpreter.bzl", "interpreter") interpreter.install( - name = "interpreter_python3_9", - python_name = "python3_9", + name = INTERPRETER_NAME_39, + python_name = PYTHON_NAME_39, +) + +# This second call is only needed if you are using mulitple different +# Python versions/interpreters. +interpreter.install( + name = INTERPRETER_NAME_310, + python_name = PYTHON_NAME_310, ) -use_repo(interpreter, "interpreter_python3_9") +use_repo(interpreter, INTERPRETER_NAME_39, INTERPRETER_NAME_310) pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") pip.parse( name = "pip", - # Intentionally set it false because the "true" case is already covered by examples/bzlmod_build_file_generation + # Intentionally set it false because the "true" case is already + # covered by examples/bzlmod_build_file_generation incompatible_generate_aliases = False, - python_interpreter_target = "@interpreter_python3_9//:python", + python_interpreter_target = "@{}//:python".format(INTERPRETER_NAME_39), requirements_lock = "//:requirements_lock.txt", requirements_windows = "//:requirements_windows.txt", ) diff --git a/examples/bzlmod/__main__.py b/examples/bzlmod/__main__.py index 099493b3c8..daf17495c2 100644 --- a/examples/bzlmod/__main__.py +++ b/examples/bzlmod/__main__.py @@ -13,6 +13,8 @@ # limitations under the License. from lib import main +import sys if __name__ == "__main__": print(main([["A", 1], ["B", 2]])) + print(sys.version) diff --git a/examples/bzlmod/other_module/MODULE.bazel b/examples/bzlmod/other_module/MODULE.bazel index 992e120760..eebfbcaa58 100644 --- a/examples/bzlmod/other_module/MODULE.bazel +++ b/examples/bzlmod/other_module/MODULE.bazel @@ -2,4 +2,56 @@ module( name = "other_module", ) +# This module is using the same version of rules_python +# that the parent module uses. bazel_dep(name = "rules_python", version = "") + +# It is not best practice to use a python.toolchian in +# a submodule. This code only exists to test that +# we support doing this. This code is only for rules_python +# testing purposes. +PYTHON_NAME_39 = "python_39" + +PYTHON_39_TOOLCHAINS = PYTHON_NAME_39 + "_toolchains" + +PYTHON_NAME_311 = "python_311" + +PYTHON_311_TOOLCHAINS = PYTHON_NAME_311 + "_toolchains" + +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain( + # This name is used in the various use_repo statements + # below, and in the local extension that is in this + # example. + name = PYTHON_NAME_39, + configure_coverage_tool = True, + python_version = "3.9", +) +python.toolchain( + # This name is used in the various use_repo statements + # below, and in the local extension that is in this + # example. + name = PYTHON_NAME_311, + configure_coverage_tool = True, + # In a submodule this is ignored + is_default = True, + python_version = "3.11", +) + +# created by the above python.toolchain calls. +use_repo( + python, + PYTHON_NAME_39, + PYTHON_39_TOOLCHAINS, + PYTHON_NAME_311, + PYTHON_311_TOOLCHAINS, +) + +# This call registers the Python toolchains. +# Note: there is work under way to move this code to within +# rules_python, and the user won't have to make this call, +# unless they are registering custom toolchains. +register_toolchains( + "@{}//:all".format(PYTHON_39_TOOLCHAINS), + "@{}//:all".format(PYTHON_311_TOOLCHAINS), +) diff --git a/examples/bzlmod/other_module/other_module/pkg/BUILD.bazel b/examples/bzlmod/other_module/other_module/pkg/BUILD.bazel index 9a130e3554..952a674d48 100644 --- a/examples/bzlmod/other_module/other_module/pkg/BUILD.bazel +++ b/examples/bzlmod/other_module/other_module/pkg/BUILD.bazel @@ -1,3 +1,4 @@ +load("@python_311//:defs.bzl", py_binary_311 = "py_binary") load("@rules_python//python:defs.bzl", "py_library") py_library( @@ -8,4 +9,15 @@ py_library( deps = ["@rules_python//python/runfiles"], ) +# This is used for testing mulitple versions of Python. This is +# used only when you need to support multiple versions of Python +# in the same project. +py_binary_311( + name = "lib_311", + srcs = ["lib.py"], + data = ["data/data.txt"], + visibility = ["//visibility:public"], + deps = ["@rules_python//python/runfiles"], +) + exports_files(["data/data.txt"]) diff --git a/examples/bzlmod/runfiles/BUILD.bazel b/examples/bzlmod/runfiles/BUILD.bazel index 3503ac3017..add56b3bd0 100644 --- a/examples/bzlmod/runfiles/BUILD.bazel +++ b/examples/bzlmod/runfiles/BUILD.bazel @@ -1,6 +1,5 @@ load("@rules_python//python:defs.bzl", "py_test") -# gazelle:ignore py_test( name = "runfiles_test", srcs = ["runfiles_test.py"], diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel index c667f1e49b..05a15cce28 100644 --- a/examples/bzlmod_build_file_generation/BUILD.bazel +++ b/examples/bzlmod_build_file_generation/BUILD.bazel @@ -7,7 +7,7 @@ # requirements. load("@bazel_gazelle//:def.bzl", "gazelle") load("@pip//:requirements.bzl", "all_whl_requirements") -load("@python3//:defs.bzl", py_test_with_transition = "py_test") +load("@python//:defs.bzl", py_test_with_transition = "py_test") load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") load("@rules_python_gazelle_plugin//:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS") diff --git a/examples/bzlmod_build_file_generation/MODULE.bazel b/examples/bzlmod_build_file_generation/MODULE.bazel index 179fe1bdea..45a1318ac8 100644 --- a/examples/bzlmod_build_file_generation/MODULE.bazel +++ b/examples/bzlmod_build_file_generation/MODULE.bazel @@ -46,7 +46,11 @@ python = use_extension("@rules_python//python/extensions:python.bzl", "python") # This name is passed into python.toolchain and it's use_repo statement. # We also use the same name for python.host_python_interpreter. -PYTHON_NAME = "python3" +PYTHON_NAME = "python" + +PYTHON_TOOLCHAINS = PYTHON_NAME + "_toolchains" + +INTERPRETER_NAME = "interpreter" # We next initialize the python toolchain using the extension. # You can set different Python versions in this block. @@ -56,6 +60,7 @@ python.toolchain( # example. name = PYTHON_NAME, configure_coverage_tool = True, + is_default = True, python_version = "3.9", ) @@ -63,12 +68,16 @@ python.toolchain( # into the scope of the current module. # All of the python3 repositories use the PYTHON_NAME as there prefix. They # are not catenated for ease of reading. -use_repo(python, PYTHON_NAME, "python3_toolchains") +use_repo( + python, + PYTHON_NAME, + PYTHON_TOOLCHAINS, +) # Register an already-defined toolchain so that Bazel can use it during # toolchain resolution. register_toolchains( - "@python3_toolchains//:all", + "@{}//:all".format(PYTHON_TOOLCHAINS), ) # The interpreter extension discovers the platform specific Python binary. @@ -76,10 +85,10 @@ register_toolchains( # pip.parse call. interpreter = use_extension("@rules_python//python/extensions:interpreter.bzl", "interpreter") interpreter.install( - name = "interpreter_python3", + name = INTERPRETER_NAME, python_name = PYTHON_NAME, ) -use_repo(interpreter, "interpreter_python3") +use_repo(interpreter, INTERPRETER_NAME) # Use the extension, pip.parse, to call the `pip_repository` rule that invokes # `pip`, with `incremental` set. The pip call accepts a locked/compiled @@ -102,7 +111,7 @@ pip.parse( # is used for both resolving dependencies and running tests/binaries. # If this isn't specified, then you'll get whatever is locally installed # on your system. - python_interpreter_target = "@interpreter_python3//:python", + python_interpreter_target = "@{}//:python".format(INTERPRETER_NAME), requirements_lock = "//:requirements_lock.txt", requirements_windows = "//:requirements_windows.txt", ) diff --git a/python/extensions/private/interpreter_hub.bzl b/python/extensions/private/interpreter_hub.bzl index f1ca670cf2..82fcbf698f 100644 --- a/python/extensions/private/interpreter_hub.bzl +++ b/python/extensions/private/interpreter_hub.bzl @@ -19,8 +19,9 @@ load("//python/private:toolchains_repo.bzl", "get_host_os_arch", "get_host_platf _build_file_for_hub_template = """ INTERPRETER_LABELS = {{ -{lines} +{interpreter_labels} }} +DEFAULT_TOOLCHAIN_NAME = "{default}" """ _line_for_hub_template = """\ @@ -35,13 +36,19 @@ def _hub_repo_impl(rctx): is_windows = (os == WINDOWS_NAME) path = "python.exe" if is_windows else "bin/python3" - lines = "\n".join([_line_for_hub_template.format( + interpreter_labels = "\n".join([_line_for_hub_template.format( name = name, platform = platform, path = path, ) for name in rctx.attr.toolchains]) - rctx.file("interpreters.bzl", _build_file_for_hub_template.format(lines = lines)) + rctx.file( + "interpreters.bzl", + _build_file_for_hub_template.format( + interpreter_labels = interpreter_labels, + default = rctx.attr.default_toolchain, + ), + ) hub_repo = repository_rule( doc = """\ @@ -50,6 +57,10 @@ and the labels to said interpreters. This map is used to by the interpreter hub """, implementation = _hub_repo_impl, attrs = { + "default_toolchain": attr.string( + doc = "Name of the default toolchain", + mandatory = True, + ), "toolchains": attr.string_list( doc = "List of the base names the toolchain repo defines.", mandatory = True, diff --git a/python/extensions/python.bzl b/python/extensions/python.bzl index 9a3d9ed959..cae1988e8a 100644 --- a/python/extensions/python.bzl +++ b/python/extensions/python.bzl @@ -17,35 +17,117 @@ load("@rules_python//python:repositories.bzl", "python_register_toolchains") load("@rules_python//python/extensions/private:interpreter_hub.bzl", "hub_repo") +# Printing a warning msg not debugging, so we have to disable +# the buildifier check. +# buildifier: disable=print +def _print_warn(msg): + print("WARNING:", msg) + +def _python_register_toolchains(toolchain_attr, version_constraint): + python_register_toolchains( + name = toolchain_attr.name, + python_version = toolchain_attr.python_version, + register_coverage_tool = toolchain_attr.configure_coverage_tool, + ignore_root_user_error = toolchain_attr.ignore_root_user_error, + set_python_version_constraint = version_constraint, + ) + def _python_impl(module_ctx): - toolchains = [] + # We collect all of the toolchain names to create + # the INTERPRETER_LABELS map. This is used + # by interpreter_extensions.bzl via the hub_repo call below. + toolchain_names = [] + + # Used to store the default toolchain name so we can pass it to the hub + default_toolchain_name = None + + # Used to store toolchains that are in sub modules. + sub_toolchains_map = {} + for mod in module_ctx.modules: for toolchain_attr in mod.tags.toolchain: - python_register_toolchains( - name = toolchain_attr.name, - python_version = toolchain_attr.python_version, - bzlmod = True, - # Toolchain registration in bzlmod is done in MODULE file - register_toolchains = False, - register_coverage_tool = toolchain_attr.configure_coverage_tool, - ignore_root_user_error = toolchain_attr.ignore_root_user_error, - ) - - # We collect all of the toolchain names to create - # the INTERPRETER_LABELS map. This is used - # by interpreter_extensions.bzl - toolchains.append(toolchain_attr.name) + # If we are in the root module we always register the toolchain. + # We wait to register the default toolchain till the end. + if mod.is_root: + toolchain_names.append(toolchain_attr.name) + + # If we have the default version or we only have one toolchain + # in the root module we set the toolchain as the default toolchain. + if toolchain_attr.is_default or len(mod.tags.toolchain) == 1: + # We have already found one default toolchain, and we can + # only have one. + if default_toolchain_name != None: + fail("""We found more than one toolchain that is marked +as the default version. Only set one toolchain with is_default set as +True. The toolchain is named: {}""".format(toolchain_attr.name)) + + # We register the default toolchain. + _python_register_toolchains(toolchain_attr, False) + default_toolchain_name = toolchain_attr.name + else: + # Always register toolchains that are in the root module. + _python_register_toolchains(toolchain_attr, version_constraint = True) + else: + # We add the toolchain to a map, and we later create the + # toolchain if the root module does not have a toolchain with + # the same name. We have to loop through all of the modules to + # ensure that we get a full list of the root toolchains. + sub_toolchains_map[toolchain_attr.name] = toolchain_attr + + # We did not find a default toolchain so we fail. + if default_toolchain_name == None: + fail("""Unable to find a default toolchain in the root module. +Please define a toolchain that has is_version set to True.""") + # Create the toolchains in the submodule(s). + for name, toolchain_attr in sub_toolchains_map.items(): + # We cannot have a toolchain in a sub module that has the same name of + # a toolchain in the root module. This will cause name clashing. + if name in toolchain_names: + _print_warn("""Not creating the toolchain from sub module, with the name {}. The root + module has a toolchain of the same name.""".format(toolchain_attr.name)) + continue + toolchain_names.append(name) + _python_register_toolchains(toolchain_attr, True) + + # Create the hub for the interpreters and the + # the default toolchain. hub_repo( name = "pythons_hub", - toolchains = toolchains, + toolchains = toolchain_names, + default_toolchain = default_toolchain_name, ) python = module_extension( - doc = "Bzlmod extension that is used to register a Python toolchain.", + doc = """Bzlmod extension that is used to register Python toolchains. +""", implementation = _python_impl, tag_classes = { "toolchain": tag_class( + doc = """Tag class used to register Python toolchains. +Use this tag class to register one or more Python toolchains. This class +is also potentially called by sub modules. The following covers different +business rules and use cases. + +Toolchains in the Root Module + +This class registers all toolchains in the root module. + +Toolchains in Sub Modules + +It will create a toolchain that is in a sub module, if the toolchain +of the same name does not exist in the root module. The extension stops name +clashing between toolchains in the root module and toolchains in sub modules. +You cannot configure more than one toolchain as the default toolchain. + +Toolchain set as the default version + +This extension will not create a toolchain that exists in a sub module, +if the sub module toolchain is marked as the default version. If you have +more than one toolchain in your root module, you need to set one of the +toolchains as the default version. If there is only one toolchain it +is set as the default toolchain. +""", attrs = { "configure_coverage_tool": attr.bool( mandatory = False, @@ -56,6 +138,10 @@ python = module_extension( doc = "Whether the check for root should be ignored or not. This causes cache misses with .pyc files.", mandatory = False, ), + "is_default": attr.bool( + mandatory = False, + doc = "Whether the toolchain is the default version", + ), "name": attr.string(mandatory = True), "python_version": attr.string(mandatory = True), }, diff --git a/python/repositories.bzl b/python/repositories.bzl index 358df4341b..4f36b12a14 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -463,7 +463,6 @@ def python_register_toolchains( register_coverage_tool = False, set_python_version_constraint = False, tool_versions = TOOL_VERSIONS, - bzlmod = False, **kwargs): """Convenience macro for users which does typical setup. @@ -486,9 +485,15 @@ def python_register_toolchains( set_python_version_constraint: When set to true, target_compatible_with for the toolchains will include a version constraint. tool_versions: a dict containing a mapping of version with SHASUM and platform info. If not supplied, the defaults in python/versions.bzl will be used. - bzlmod: Whether this rule is being run under a bzlmod module extension. **kwargs: passed to each python_repositories call. """ + + # If we have @@ we have bzlmod + bzlmod = str(Label("//:unused")).startswith("@@") + if bzlmod: + # you cannot used native.register_toolchains when using bzlmod. + register_toolchains = False + base_url = kwargs.pop("base_url", DEFAULT_RELEASE_BASE_URL) if python_version in MINOR_MAPPING: