Merge remote-tracking branch 'origin/main' into siddharthab/main

bazelbuild · Dec 20, 2023 · 2a3adbd · 2a3adbd
2 parents 91455ad + 509b02f
commit 2a3adbd
Show file tree

Hide file tree

Showing 98 changed files with 2,911 additions and 526 deletions.
diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml
@@ -21,8 +21,8 @@ buildifier:
 .minimum_supported_version: &minimum_supported_version
   # For testing minimum supported version.
   # NOTE: Keep in sync with //:version.bzl
-  bazel: 5.4.0
-  skip_in_bazel_downstream_pipeline: "Bazel 5 required"
+  bazel: 6.2.0
+  skip_in_bazel_downstream_pipeline: "Bazel 6 required"
 .minimum_supported_bzlmod_version: &minimum_supported_bzlmod_version
   bazel: 6.2.0 # test minimum supported version of bazel for bzlmod tests
 .reusable_config: &reusable_config
@@ -64,10 +64,7 @@ buildifier:
     - //tests:version_3_9_test
     - //tests:version_default_test
 .pystar_base: &pystar_base
-  # TODO: Change to "7.x" once Bazel 7 is available
-  # https://github.com/bazelbuild/bazel/commit/f3aafea59ae021c6a12086cb2cd34c5fa782faf1
-  # is available in rolling.
-  bazel: "last_rc"
+  bazel: "7.x"
   environment:
     RULES_PYTHON_ENABLE_PYSTAR: "1"
   test_flags:

diff --git a/.bazelignore b/.bazelignore
@@ -13,6 +13,10 @@ bazel-testlogs
 examples/bzlmod/bazel-bzlmod
 examples/bzlmod/other_module/bazel-other_module
 examples/bzlmod_build_file_generation/bazel-bzlmod_build_file_generation
+examples/multi_python_versions/bazel-multi_python_versions
 examples/pip_parse/bazel-pip_parse
+examples/pip_parse_vendored/bazel-pip_parse_vendored
 examples/py_proto_library/bazel-py_proto_library
+tests/compile_pip_requirements/bazel-compile_pip_requirements
 tests/ignore_root_user_error/bazel-ignore_root_user_error
+tests/pip_repository_entry_points/bazel-pip_repository_entry_points
diff --git a/.bazelrc b/.bazelrc
@@ -3,8 +3,8 @@
 # This lets us glob() up all the files inside the examples to make them inputs to tests
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points
 
 test --test_output=errors
 
@@ -25,5 +25,8 @@ startup --windows_enable_symlinks
 common --noexperimental_enable_bzlmod
 
 # Additional config to use for readthedocs builds.
-# See .readthedocs.yml for additional flags
+# See .readthedocs.yml for additional flags that can only be determined from
+# the runtime environment.
 build:rtd --stamp
+# Some bzl files contain repos only available under bzlmod
+build:rtd --enable_bzlmod
diff --git a/.bazelversion b/.bazelversion
@@ -1 +1 @@
-6.2.0
+7.0.0
diff --git a/.github/workflows/create_archive_and_notes.sh b/.github/workflows/create_archive_and_notes.sh
@@ -37,7 +37,8 @@ bazel_dep(name = "rules_python", version = "${TAG}")
 pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
 
 pip.parse(
-    name = "pip",
+    hub_name = "pip",
+    python_version = "3.11",
     requirements_lock = "//:requirements_lock.txt",
 )
 

diff --git a/BUILD.bazel b/BUILD.bazel
@@ -24,6 +24,11 @@ exports_files([
     "version.bzl",
 ])
 
+exports_files(
+    glob(["*.md"]),
+    visibility = ["//docs:__subpackages__"],
+)
+
 filegroup(
     name = "distribution",
     srcs = [

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -36,6 +36,21 @@ A brief description of the categories of changes:
 * (gazelle) The gazelle plugin helper was not working with Python toolchains 3.11
   and above due to a bug in the helper components not being on PYTHONPATH.
 
+* (pip_parse) The repositories created by `whl_library` can now parse the `whl`
+  METADATA and generate dependency closures irrespective of the host platform
+  the generation is executed on. This can be turned on by supplying
+  `experimental_target_platforms = ["all"]` to the `pip_parse` or the `bzlmod`
+  equivalent. This may help in cases where fetching wheels for a different
+  platform using `download_only = True` feature.
+* (bzlmod pip.parse) The `pip.parse(python_interpreter)` arg now works for
+  specifying a local system interpreter.
+* (bzlmod pip.parse) Requirements files with duplicate entries for the same
+  package (e.g. one for the package, one for an extra) now work.
+
+### Added
+
+* (docs) bzlmod extensions are now documented on rules-python.readthedocs.io
+
 [0.XX.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.XX.0
 
 ## [0.27.0] - 2023-11-16

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -65,10 +65,6 @@ and setup. Subsequent runs will be faster, but there are many tests, and some of
 them are slow. If you're working on a particular area of code, you can run just
 the tests in those directories instead, which can speed up your edit-run cycle.
 
-Note that there are tests to verify generated documentation is correct -- if
-you're modifying the signature of a public function, these tests will likely
-fail and you'll need to [regenerate the api docs](#documentation).
-
 ## Formatting
 
 Starlark files should be formatted by
@@ -150,17 +146,6 @@ For the full details of types, see
 Some checked-in files are generated and need to be updated when a new PR is
 merged.
 
-### Documentation
-
-To regenerate the content under the `docs/` directory, run this command:
-
-```shell
-bazel run //docs:update
-```
-
-This needs to be done whenever the docstrings in the corresponding .bzl files
-are changed; a test failure will remind you to run this command when needed.
-
 ## Core rules
 
 The bulk of this repo is owned and maintained by the Bazel Python community.

diff --git a/MODULE.bazel b/MODULE.bazel
@@ -56,12 +56,12 @@ use_repo(python, "pythons_hub")
 register_toolchains("@pythons_hub//:all")
 
 # ===== DEV ONLY SETUP =====
-docs_pip = use_extension(
+dev_pip = use_extension(
     "//python/extensions:pip.bzl",
     "pip",
     dev_dependency = True,
 )
-docs_pip.parse(
+dev_pip.parse(
     experimental_requirement_cycles = {
         "sphinx": [
             "sphinx",
@@ -72,7 +72,7 @@ docs_pip.parse(
             "sphinxcontrib-applehelp",
         ],
     },
-    hub_name = "docs_deps",
+    hub_name = "dev_pip",
     python_version = "3.11",
     requirements_darwin = "//docs/sphinx:requirements_darwin.txt",
     requirements_lock = "//docs/sphinx:requirements_linux.txt",

diff --git a/WORKSPACE b/WORKSPACE
@@ -109,7 +109,7 @@ install_deps()
 # Install sphinx for doc generation.
 
 pip_parse(
-    name = "docs_deps",
+    name = "dev_pip",
     experimental_requirement_cycles = {
         "sphinx": [
             "sphinx",
@@ -126,7 +126,7 @@ pip_parse(
     requirements_lock = "//docs/sphinx:requirements_linux.txt",
 )
 
-load("@docs_deps//:requirements.bzl", docs_install_deps = "install_deps")
+load("@dev_pip//:requirements.bzl", docs_install_deps = "install_deps")
 
 docs_install_deps()
 
@@ -140,3 +140,9 @@ http_file(
         "https://files.pythonhosted.org/packages/50/67/3e966d99a07d60a21a21d7ec016e9e4c2642a86fea251ec68677daf71d4d/numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
     ],
 )
+
+# rules_proto expects //external:python_headers to point at the python headers.
+bind(
+    name = "python_headers",
+    actual = "//python/cc:current_py_cc_headers",
+)
diff --git a/docs/sphinx/BUILD.bazel b/docs/sphinx/BUILD.bazel
@@ -12,8 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("@docs_deps//:requirements.bzl", "requirement")
-load("@rules_python//python:pip.bzl", "compile_pip_requirements")
+load("@dev_pip//:requirements.bzl", "requirement")
+load("//python:pip.bzl", "compile_pip_requirements")
+load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")  # buildifier: disable=bzl-visibility
+load("//python/private:util.bzl", "IS_BAZEL_7_OR_HIGHER")  # buildifier: disable=bzl-visibility
 load("//sphinxdocs:readthedocs.bzl", "readthedocs_install")
 load("//sphinxdocs:sphinx.bzl", "sphinx_build_binary", "sphinx_docs", "sphinx_inventory")
 load("//sphinxdocs:sphinx_stardoc.bzl", "sphinx_stardocs")
@@ -55,6 +57,10 @@ sphinx_docs(
     formats = [
         "html",
     ],
+    renamed_srcs = {
+        "//:CHANGELOG.md": "changelog.md",
+        "//:CONTRIBUTING.md": "contributing.md",
+    },
     sphinx = ":sphinx-build",
     strip_prefix = package_name() + "/",
     tags = ["docs"],
@@ -79,7 +85,13 @@ sphinx_stardocs(
         "api/entry_points/py_console_script_binary.md": "//python/entry_points:py_console_script_binary_bzl",
         "api/packaging.md": "//python:packaging_bzl",
         "api/pip.md": "//python:pip_bzl",
-    },
+    } | ({
+        # Bazel 6 + Stardoc isn't able to parse something about the python bzlmod extension
+        "api/extensions/python.md": "//python/extensions:python_bzl",
+    } if IS_BAZEL_7_OR_HIGHER else {}) | ({
+        # This depends on @pythons_hub, which is only created under bzlmod
+        "api/extensions/pip.md": "//python/extensions:pip_bzl",
+    } if BZLMOD_ENABLED else {}),
     footer = "_stardoc_footer.md",
     tags = ["docs"],
     target_compatible_with = _TARGET_COMPATIBLE_WITH,

diff --git a/docs/sphinx/_stardoc_footer.md b/docs/sphinx/_stardoc_footer.md
@@ -7,6 +7,8 @@
 [`Label`]: https://bazel.build/rules/lib/Label
 [`list`]: https://bazel.build/rules/lib/list
 [`str`]: https://bazel.build/rules/lib/string
+[str]: https://bazel.build/rules/lib/string
+[`int`]: https://bazel.build/rules/lib/int
 [`struct`]: https://bazel.build/rules/lib/builtins/struct
 [`Target`]: https://bazel.build/rules/lib/Target
 [target-name]: https://bazel.build/concepts/labels#target-names

diff --git a/docs/sphinx/_static/css/custom.css b/docs/sphinx/_static/css/custom.css
@@ -12,8 +12,17 @@
   border-bottom: thin solid grey;
   padding-left: 0.5ex;
 }
+.starlark-object h3 {
+  background-color: #e7f2fa;
+  padding-left: 0.5ex;
+}
+
+.starlark-module-extension-tag-class h3 {
+  background-color: #add8e6;
+  padding-left: 0.5ex;
+}
 
-.starlark-object>p, .starlark-object>dl {
+.starlark-object>p, .starlark-object>dl, .starlark-object>section>* {
   /* Prevent the words from touching the border line */
   padding-left: 0.5ex;
 }

diff --git a/docs/sphinx/index.md b/docs/sphinx/index.md
@@ -60,6 +60,8 @@ pypi-dependencies
 pip
 coverage
 gazelle
+Contributing <contributing>
+Changelog <changelog>
 api/index
 glossary
 genindex

diff --git a/docs/sphinx/pyproject.toml b/docs/sphinx/pyproject.toml
@@ -9,4 +9,5 @@ dependencies = [
     "myst-parser",
     "sphinx_rtd_theme",
     "readthedocs-sphinx-ext",
+    "absl-py",
 ]
diff --git a/docs/sphinx/readthedocs_build.sh b/docs/sphinx/readthedocs_build.sh
@@ -14,6 +14,7 @@ extra_env+=("--//sphinxdocs:extra_env=HOSTNAME=$HOSTNAME")
 
 set -x
 bazel run \
+  --config=rtd \
   "--//sphinxdocs:extra_defines=version=$READTHEDOCS_VERSION" \
   "${extra_env[@]}" \
   //docs/sphinx:readthedocs_install
diff --git a/docs/sphinx/requirements_linux.txt b/docs/sphinx/requirements_linux.txt
@@ -4,6 +4,10 @@
 #
 #    bazel run //docs/sphinx:requirements.update
 #
+absl-py==2.0.0 \
+    --hash=sha256:9a28abb62774ae4e8edbe2dd4c49ffcd45a6a848952a5eccc6a49f3f0fc1e2f3 \
+    --hash=sha256:d9690211c5fcfefcdd1a45470ac2b5c5acd45241c3af71eed96bc5441746c0d5
+    # via rules-python-docs (docs/sphinx/pyproject.toml)
 alabaster==0.7.13 \
     --hash=sha256:1ee19aca801bbabb5ba3f5f258e4422dfa86f82f3e9cefb0859b283cdd7f62a3 \
     --hash=sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2

diff --git a/examples/BUILD.bazel b/examples/BUILD.bazel
@@ -11,57 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-load("//tools/bazel_integration_test:bazel_integration_test.bzl", "bazel_integration_test")
 
 package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])  # Apache 2.0
-
-bazel_integration_test(
-    name = "build_file_generation_example",
-    timeout = "long",
-)
-
-bazel_integration_test(
-    name = "bzlmod_build_file_generation_example",
-    timeout = "long",
-)
-
-bazel_integration_test(
-    name = "pip_parse_example",
-    timeout = "long",
-)
-
-bazel_integration_test(
-    name = "pip_parse_vendored_example",
-    timeout = "long",
-    tags = ["fix-windows"],
-)
-
-bazel_integration_test(
-    name = "pip_repository_annotations_example",
-    timeout = "long",
-)
-
-bazel_integration_test(
-    name = "py_proto_library_example",
-    timeout = "long",
-)
-
-bazel_integration_test(
-    name = "py_proto_library_example_bzlmod",
-    timeout = "long",
-    bzlmod = True,
-    dirname = "py_proto_library",
-)
-
-bazel_integration_test(
-    name = "multi_python_versions_example",
-    timeout = "long",
-)
-
-bazel_integration_test(
-    name = "bzlmod_example",
-    bzlmod = True,
-    override_bazel_version = "6.2.0",
-)
diff --git a/examples/bzlmod/BUILD.bazel b/examples/bzlmod/BUILD.bazel
@@ -25,6 +25,7 @@ compile_pip_requirements_3_9(
 # with pip-compile.
 compile_pip_requirements_3_10(
     name = "requirements_3_10",
+    timeout = "moderate",
     src = "requirements.in",
     requirements_txt = "requirements_lock_3_10.txt",
     requirements_windows = "requirements_windows_3_10.txt",

diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel
@@ -98,6 +98,13 @@ pip.parse(
             "sphinxcontrib-serializinghtml",
         ],
     },
+    # You can use one of the values below to specify the target platform
+    # to generate the dependency graph for.
+    experimental_target_platforms = [
+        "all",
+        "linux_*",
+        "host",
+    ],
     hub_name = "pip",
     python_version = "3.9",
     requirements_lock = "//:requirements_lock_3_9.txt",
@@ -121,6 +128,13 @@ pip.parse(
             "sphinxcontrib-serializinghtml",
         ],
     },
+    # You can use one of the values below to specify the target platform
+    # to generate the dependency graph for.
+    experimental_target_platforms = [
+        "all",
+        "linux_*",
+        "host",
+    ],
     hub_name = "pip",
     python_version = "3.10",
     requirements_lock = "//:requirements_lock_3_10.txt",
@@ -153,3 +167,14 @@ local_path_override(
     module_name = "other_module",
     path = "other_module",
 )
+
+# =====
+# Config for testing duplicate packages in requirements
+# =====
+#
+pip.parse(
+    hub_name = "dupe_requirements",
+    python_version = "3.9",  # Must match whatever is marked is_default=True
+    requirements_lock = "//tests/dupe_requirements:requirements.txt",
+)
+use_repo(pip, "dupe_requirements")
diff --git a/examples/bzlmod/requirements.in b/examples/bzlmod/requirements.in
@@ -1,4 +1,4 @@
---extra-index-url https://pypi.python.org/simple/
+--extra-index-url https://pypi.org/simple/
 
 wheel
 websockets