Add --pex-executable-search-path (#10489)

Previously, `--python-setup-interpreter-search-paths` would be used both to find Python interpreters and to set the $PATH for the running PEX subprocess. This resulted in many awkward/impossible set ups.

Switch to using PEX_PYTHON_PATH to both discover a bootstrap
interpreter and steer runtime interpreter selection. Introduce a
find_binary rule to facilitate this that uses ~portable bash for now.
Also introduce a PexRuntimeEnvironment to steer bootstrap interpreter
probing and allow specification of the PATH that should be exposed to
the PEX runtime environment.

Fixes #9760.

[ci skip-rust]
[ci skip-build-wheels]

.travis.yml

@@ -15,6 +15,7 @@ deploy:
 env:
   global:
   - PANTS_CONFIG_FILES="${TRAVIS_BUILD_DIR}/pants.travis-ci.toml"
+  - PANTS_DYNAMIC_UI=false
   - LC_ALL="en_US.UTF-8"
   - AWS_BUCKET=ci-public.pantsbuild.org
   - BOOTSTRAPPED_PEX_KEY_PREFIX=daily/${TRAVIS_BUILD_NUMBER}/${TRAVIS_BUILD_ID}/pants.pex
@@ -327,7 +328,6 @@ jobs:
     env:
     - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux
     - PANTS_REMOTE_CA_CERTS_PATH=/usr/lib/google-cloud-sdk/lib/third_party/grpc/_cython/_credentials/roots.pem
-    - PANTS_NATIVE_BUILD_STEP_CPP_COMPILE_SETTINGS_DEFAULT_COMPILER_OPTION_SETS="[]"
     - CACHE_NAME=lint.py37
     language: python
     name: Self-checks and lint (Python 3.7)
@@ -487,7 +487,6 @@ jobs:
     env:
     - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux
     - PANTS_REMOTE_CA_CERTS_PATH=/usr/lib/google-cloud-sdk/lib/third_party/grpc/_cython/_credentials/roots.pem
-    - PANTS_NATIVE_BUILD_STEP_CPP_COMPILE_SETTINGS_DEFAULT_COMPILER_OPTION_SETS="[]"
     - CACHE_NAME=python_tests.py37
     language: python
     name: Python tests (Python 3.7)

build-support/bin/generate_travis_yml.py

@@ -67,6 +67,7 @@ def all_entries(cls) -> List[Dict[str, str]]:
 
 GLOBAL_ENV_VARS = [
     'PANTS_CONFIG_FILES="${TRAVIS_BUILD_DIR}/pants.travis-ci.toml"',
+    "PANTS_DYNAMIC_UI=false",
     'LC_ALL="en_US.UTF-8"',
     "AWS_BUCKET=ci-public.pantsbuild.org",
     # The ci-public.pantsbuild.org bucket has expiration policies set up for key prefixes
@@ -355,11 +356,6 @@ def linux_shard(
             "PANTS_REMOTE_CA_CERTS_PATH=/usr/lib/google-cloud-sdk/lib/third_party/grpc/_cython/_credentials/roots.pem",
         ]
         setup = {**setup, **CACHE_PANTS_RUN}
-        if python_version.is_py37:
-            # 3.7.2 for Linux uses the new C++ ABI, which may be an error.
-            setup["env"].append(
-                'PANTS_NATIVE_BUILD_STEP_CPP_COMPILE_SETTINGS_DEFAULT_COMPILER_OPTION_SETS="[]"'
-            )
     if use_docker:
         setup["services"] = ["docker"]
     return setup
@@ -449,7 +445,7 @@ def osx_shard(
 
 def _bootstrap_command(*, python_version: PythonVersion) -> str:
     return (
-        f"./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version "
+        "./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version "
         f"{python_version.decimal} --aws-bucket ${{AWS_BUCKET}} --native-engine-so-key-prefix "
         "${NATIVE_ENGINE_SO_KEY_PREFIX} --pex-key "
         "${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}"

pants.remote.toml

@@ -32,13 +32,25 @@ process_execution_speculation_strategy = "none"
 # p95 of RBE appears to be ~ 2 seconds, but we need to factor in local queue time which can be much longer, but no metrics yet.
 process_execution_speculation_delay = 15
 
+[pex]
+# TODO(#7735): This config is not ideal, that we must specify the PATH for both local and remote
+# platforms. This should be replaced by a proper mechanism to differentiate between the two.
+executable_search_paths = [
+  # These are the interpreter paths we set up on the remote container. We need to specify these
+  # because many of our tests for the Python backend need to discover Python interpreters.
+  "/pyenv-docker-build/versions/3.7.3/bin:/pyenv-docker-build/versions/3.6.8/bin:/pyenv-docker-build/versions/2.7.15/bin",
+  # The remote container has binaries like `ld` and `gcc` in /usr/bin.
+  "/usr/bin",
+  # We include the host PATH so that speculation still works.
+  '<PATH>',
+]
+
 [python-setup]
 # TODO(#7735): This config is not ideal, that we must specify the PATH for both local and remote
 # platforms. This should be replaced by a proper mechanism to differentiate between the two.
 interpreter_search_paths = [
-  # These are the interpreter paths we set up on the remote container, plus `/usr/bin`, so that
-  # pip can find `ld` if necessary.
-  "/pyenv-docker-build/versions/3.7.3/bin:/pyenv-docker-build/versions/3.6.8/bin:/pyenv-docker-build/versions/2.7.15/bin:/usr/bin",
+  # These are the interpreter paths we set up on the remote container.
+  "/pyenv-docker-build/versions/3.7.3/bin:/pyenv-docker-build/versions/3.6.8/bin:/pyenv-docker-build/versions/2.7.15/bin",
   # We include the host PATH and PEXRC values so that speculation still works.
   # NOTE: These come after the remote paths. Putting them before the remote paths means generic
   # bin dirs like /usr/bin will be on the PATH ahead of the pyenv dirs we actually want to use

pants.travis-ci.toml

@@ -1,4 +1,3 @@
-# Overrides for TravisCI runs.
 [DEFAULT]
 # If we use typical default process parallelism tied to core count, we see too many cores under
 # travis and either get oomkilled from launching too many processes with too much total memory
@@ -25,4 +24,4 @@ resolver_jobs = 2
 use_coverage = true
 
 [coverage-py]
-report=["raw"]
+report = ["raw"]

src/python/pants/backend/awslambda/python/awslambda_python_rules.py

@@ -13,6 +13,7 @@
     PythonAwsLambdaRuntime,
 )
 from pants.backend.python.rules import download_pex_bin, pex, pex_from_targets, python_sources
+from pants.backend.python.rules.hermetic_pex import PexEnvironment
 from pants.backend.python.rules.pex import (
     Pex,
     PexInterpreterConstraints,
@@ -32,7 +33,6 @@
 from pants.engine.process import Process, ProcessResult
 from pants.engine.rules import Get, collect_rules, rule
 from pants.engine.unions import UnionRule
-from pants.python.python_setup import PythonSetup
 
 
 @dataclass(frozen=True)
@@ -52,7 +52,7 @@ class LambdexSetup:
 async def create_python_awslambda(
     field_set: PythonAwsLambdaFieldSet,
     lambdex_setup: LambdexSetup,
-    python_setup: PythonSetup,
+    pex_environment: PexEnvironment,
     subprocess_environment: SubprocessEnvironment,
 ) -> CreatedAWSLambda:
     # Lambdas typically use the .zip suffix, so we use that instead of .pex.
@@ -91,7 +91,7 @@ async def create_python_awslambda(
     # NB: Lambdex modifies its input pex in-place, so the input file is also the output file.
     lambdex_args = ("build", "-e", field_set.handler.value, pex_filename)
     process = lambdex_setup.requirements_pex.create_process(
-        python_setup=python_setup,
+        pex_environment=pex_environment,
         subprocess_environment=subprocess_environment,
         pex_path="./lambdex.pex",
         pex_args=lambdex_args,

src/python/pants/backend/python/lint/bandit/rules.py

@@ -6,6 +6,7 @@
 
 from pants.backend.python.lint.bandit.subsystem import Bandit
 from pants.backend.python.rules import download_pex_bin, pex
+from pants.backend.python.rules.hermetic_pex import PexEnvironment
 from pants.backend.python.rules.pex import (
     Pex,
     PexInterpreterConstraints,
@@ -80,7 +81,7 @@ async def bandit_lint_partition(
     partition: BanditPartition,
     bandit: Bandit,
     lint_subsystem: LintSubsystem,
-    python_setup: PythonSetup,
+    pex_environment: PexEnvironment,
     subprocess_environment: SubprocessEnvironment,
 ) -> LintResult:
     requirements_pex_request = Get(
@@ -139,7 +140,7 @@ async def bandit_lint_partition(
         output_file=report_path.name if report_path else None,
     )
     process = requirements_pex.create_process(
-        python_setup=python_setup,
+        pex_environment=pex_environment,
         subprocess_environment=subprocess_environment,
         pex_path="./bandit.pex",
         pex_args=args,

src/python/pants/backend/python/lint/black/rules.py

@@ -9,6 +9,7 @@
 from pants.backend.python.lint.black.subsystem import Black
 from pants.backend.python.lint.python_fmt import PythonFmtRequest
 from pants.backend.python.rules import download_pex_bin, pex
+from pants.backend.python.rules.hermetic_pex import PexEnvironment
 from pants.backend.python.rules.pex import (
     Pex,
     PexInterpreterConstraints,
@@ -31,7 +32,6 @@
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
 from pants.engine.target import FieldSetWithOrigin
 from pants.engine.unions import UnionRule
-from pants.python.python_setup import PythonSetup
 from pants.util.strutil import pluralize
 
 
@@ -81,7 +81,7 @@ def generate_args(
 async def setup(
     setup_request: SetupRequest,
     black: Black,
-    python_setup: PythonSetup,
+    pex_environment: PexEnvironment,
     subprocess_environment: SubprocessEnvironment,
 ) -> Setup:
     requirements_pex_request = Get(
@@ -136,7 +136,7 @@ async def setup(
     )
 
     process = requirements_pex.create_process(
-        python_setup=python_setup,
+        pex_environment=pex_environment,
         subprocess_environment=subprocess_environment,
         pex_path="./black.pex",
         pex_args=generate_args(

src/python/pants/backend/python/lint/docformatter/rules.py

@@ -7,6 +7,7 @@
 from pants.backend.python.lint.docformatter.subsystem import Docformatter
 from pants.backend.python.lint.python_fmt import PythonFmtRequest
 from pants.backend.python.rules import download_pex_bin, pex
+from pants.backend.python.rules.hermetic_pex import PexEnvironment
 from pants.backend.python.rules.pex import (
     Pex,
     PexInterpreterConstraints,
@@ -29,7 +30,6 @@
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
 from pants.engine.target import FieldSetWithOrigin
 from pants.engine.unions import UnionRule
-from pants.python.python_setup import PythonSetup
 from pants.util.strutil import pluralize
 
 
@@ -70,7 +70,7 @@ def generate_args(
 async def setup(
     setup_request: SetupRequest,
     docformatter: Docformatter,
-    python_setup: PythonSetup,
+    pex_environment: PexEnvironment,
     subprocess_environment: SubprocessEnvironment,
 ) -> Setup:
     requirements_pex_request = Get(
@@ -115,7 +115,7 @@ async def setup(
     )
 
     process = requirements_pex.create_process(
-        python_setup=python_setup,
+        pex_environment=pex_environment,
         subprocess_environment=subprocess_environment,
         pex_path="./docformatter.pex",
         pex_args=generate_args(

src/python/pants/backend/python/lint/flake8/rules.py

@@ -6,6 +6,7 @@
 
 from pants.backend.python.lint.flake8.subsystem import Flake8
 from pants.backend.python.rules import download_pex_bin, pex
+from pants.backend.python.rules.hermetic_pex import PexEnvironment
 from pants.backend.python.rules.pex import (
     Pex,
     PexInterpreterConstraints,
@@ -80,7 +81,7 @@ async def flake8_lint_partition(
     partition: Flake8Partition,
     flake8: Flake8,
     lint_subsystem: LintSubsystem,
-    python_setup: PythonSetup,
+    pex_environment: PexEnvironment,
     subprocess_environment: SubprocessEnvironment,
 ) -> LintResult:
     requirements_pex_request = Get(
@@ -139,7 +140,7 @@ async def flake8_lint_partition(
         output_file=report_path.name if report_path else None,
     )
     process = requirements_pex.create_process(
-        python_setup=python_setup,
+        pex_environment=pex_environment,
         subprocess_environment=subprocess_environment,
         pex_path="./flake8.pex",
         pex_args=args,

src/python/pants/backend/python/lint/isort/rules.py

@@ -7,6 +7,7 @@
 from pants.backend.python.lint.isort.subsystem import Isort
 from pants.backend.python.lint.python_fmt import PythonFmtRequest
 from pants.backend.python.rules import download_pex_bin, pex
+from pants.backend.python.rules.hermetic_pex import PexEnvironment
 from pants.backend.python.rules.pex import (
     Pex,
     PexInterpreterConstraints,
@@ -36,7 +37,6 @@
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
 from pants.engine.target import FieldSetWithOrigin
 from pants.engine.unions import UnionRule
-from pants.python.python_setup import PythonSetup
 from pants.util.strutil import pluralize
 
 
@@ -80,7 +80,7 @@ def generate_args(
 async def setup(
     setup_request: SetupRequest,
     isort: Isort,
-    python_setup: PythonSetup,
+    pex_environment: PexEnvironment,
     subprocess_environment: SubprocessEnvironment,
 ) -> Setup:
     requirements_pex_request = Get(
@@ -140,7 +140,7 @@ async def setup(
     )
 
     process = requirements_pex.create_process(
-        python_setup=python_setup,
+        pex_environment=pex_environment,
         subprocess_environment=subprocess_environment,
         pex_path="./isort.pex",
         pex_args=generate_args(

src/python/pants/backend/python/lint/pylint/rules.py

@@ -8,6 +8,7 @@
 
 from pants.backend.python.lint.pylint.subsystem import Pylint
 from pants.backend.python.rules import download_pex_bin, pex, python_sources
+from pants.backend.python.rules.hermetic_pex import PexEnvironment
 from pants.backend.python.rules.pex import (
     Pex,
     PexInterpreterConstraints,
@@ -110,7 +111,7 @@ def generate_args(*, specified_source_files: SourceFiles, pylint: Pylint) -> Tup
 async def pylint_lint_partition(
     partition: PylintPartition,
     pylint: Pylint,
-    python_setup: PythonSetup,
+    pex_environment: PexEnvironment,
     subprocess_environment: SubprocessEnvironment,
 ) -> LintResult:
     # We build one PEX with Pylint requirements and another with all direct 3rd-party dependencies.
@@ -231,7 +232,7 @@ async def pylint_lint_partition(
     )
 
     process = pylint_runner_pex.create_process(
-        python_setup=python_setup,
+        pex_environment=pex_environment,
         subprocess_environment=subprocess_environment,
         pex_path="./pylint_runner.pex",
         env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)},

src/python/pants/backend/python/rules/coverage.py

@@ -7,6 +7,7 @@
 from pathlib import PurePath
 from typing import List, Optional, Sequence, Tuple, cast
 
+from pants.backend.python.rules.hermetic_pex import PexEnvironment
 from pants.backend.python.rules.pex import (
     Pex,
     PexInterpreterConstraints,
@@ -44,7 +45,6 @@
 from pants.engine.target import TransitiveTargets
 from pants.engine.unions import UnionRule
 from pants.option.custom_types import file_option
-from pants.python.python_setup import PythonSetup
 
 
 """
@@ -215,7 +215,7 @@ class MergedCoverageData:
 async def merge_coverage_data(
     data_collection: PytestCoverageDataCollection,
     coverage_setup: CoverageSetup,
-    python_setup: PythonSetup,
+    pex_environment: PexEnvironment,
     subprocess_environment: SubprocessEnvironment,
 ) -> MergedCoverageData:
     if len(data_collection) == 1:
@@ -233,7 +233,7 @@ async def merge_coverage_data(
         input_digest=input_digest,
         output_files=(".coverage",),
         description=f"Merge {len(prefixes)} Pytest coverage reports.",
-        python_setup=python_setup,
+        pex_environment=pex_environment,
         subprocess_environment=subprocess_environment,
     )
     result = await Get(ProcessResult, Process, process)
@@ -247,7 +247,7 @@ async def generate_coverage_reports(
     coverage_config: CoverageConfig,
     coverage_subsystem: CoverageSubsystem,
     transitive_targets: TransitiveTargets,
-    python_setup: PythonSetup,
+    pex_environment: PexEnvironment,
     subprocess_environment: SubprocessEnvironment,
 ) -> CoverageReports:
     """Takes all Python test results and generates a single coverage report."""
@@ -292,7 +292,7 @@ async def generate_coverage_reports(
                 output_directories=("htmlcov",) if report_type == CoverageReportType.HTML else None,
                 output_files=("coverage.xml",) if report_type == CoverageReportType.XML else None,
                 description=f"Generate Pytest {report_type.report_name} coverage report.",
-                python_setup=python_setup,
+                pex_environment=pex_environment,
                 subprocess_environment=subprocess_environment,
             )
         )