Allow Pants to run with Python 3 via ./pants3 script

.travis.yml

@@ -43,7 +43,8 @@ native_engine_cache_config: &native_engine_cache_config
     timeout: 500
     directories:
       - ${HOME}/.cache/pants/rust/cargo
-      - build-support/pants_dev_deps.venv
+      - build-support/pants_dev_deps.py2.venv
+      - build-support/pants_dev_deps.py3.venv
       - src/rust/engine/target
 
 # Travis cache config for jobs that run a bootstrapped pants.pex.

build-support/pants_venv

@@ -12,7 +12,8 @@ REQUIREMENTS=(
 )
 
 function venv_dir() {
-  echo ${REPO_ROOT}/build-support/pants_dev_deps.venv
+  py_version="py2"; [ "${PANTS_USE_PYTHON3}" = true ] && py_version="py3"
+  echo ${REPO_ROOT}/build-support/pants_dev_deps.${py_version}.venv
 }
 
 function activate_venv() {
@@ -21,6 +22,15 @@ function activate_venv() {
 
 function create_venv() {
   rm -rf "$(venv_dir)"
+  # If `pants_dev_deps.venv` still exists, delete both its legacy folder and the cached
+  # Python interpreter folder, which contains problematic symlinks.
+  # Note we only perform these removals for the first time, to avoid continually deleting
+  # the cache when switching between using `pants_dev_deps.py2.venv` and `pants_dev_deps.py3.venv`.
+  legacy_venv_dir="${REPO_ROOT}/build-support/pants_dev_deps.venv"
+  if [ -d "${legacy_venv_dir}" ]; then
+    rm -rf "${legacy_venv_dir}"
+    rm -rf "${HOME}/.cache/pants/python_cache/interpreters"
+  fi
   "${REPO_ROOT}/build-support/virtualenv" "$(venv_dir)"
 }
 

build-support/python/clean.sh

@@ -3,5 +3,6 @@
 PANTS_BASE=$(dirname $0)/../..
 rm -rf ${HOME}/.pex
 rm -rf ${PANTS_BASE}/build-support/pants_dev_deps.venv
+rm -rf ${PANTS_BASE}/build-support/pants_dev_deps.py{2,3}.venv
 rm -rf ${PANTS_BASE}/.pants.d
 find ${PANTS_BASE} -name '*.pyc' | xargs rm -f

build-support/travis/travis.yml.mustache

@@ -39,7 +39,8 @@ native_engine_cache_config: &native_engine_cache_config
     timeout: 500
     directories:
       - ${HOME}/.cache/pants/rust/cargo
-      - build-support/pants_dev_deps.venv
+      - build-support/pants_dev_deps.py2.venv
+      - build-support/pants_dev_deps.py3.venv
       - src/rust/engine/target
 
 # Travis cache config for jobs that run a bootstrapped pants.pex.

build-support/virtualenv

@@ -14,10 +14,11 @@ fi
 source ${REPO_ROOT}/build-support/common.sh
 
 if [[ -z "${PY}" ]]; then
-  if which python2.7 >/dev/null; then
-    PY=`which python2.7`
+  python_bin_name="python2.7"; [ "${PANTS_USE_PYTHON3}" = true ] && python_bin_name="python3"
+  if which "${python_bin_name}" >/dev/null; then
+    PY=`which ${python_bin_name}`
   else
-    die 'No python2.7 interpreter found on the path.  Python will not work!'
+    die "No ${python_bin_name} interpreter found on the path. Python will not work!"
   fi
 fi
 

pants.ini

@@ -369,10 +369,11 @@ verify_commit: False
 
 
 [python-setup]
-# We only support pants running under 2.7 for now with 3.3+ support to be added later.
-# Any example or test targets that are meant to test interpreters outside pants own
-# acceptable set should specify an explicit compatibility constraint.
-interpreter_constraints: ["CPython>=2.7,<3"]
+# We will support Python 2 until release 1.16, at which we will drop Python 2 support and
+# require Python 3.6+ to run Pants.
+# Note this does not mean client code has to use these specified versions, only that
+# an appropriate interpreter must be discoverable.
+interpreter_constraints: ["CPython>=2.7,<3","CPython>=3.6,<4"]
 interpreter_cache_dir: %(pants_bootstrapdir)s/python_cache/interpreters
 resolver_cache_dir: %(pants_bootstrapdir)s/python_cache/requirements
 

pants3

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+# This bootstrap script invokes Pants using a Python 3 interpreter.
+
+# Use Py3 under-the-hood
+export PANTS_USE_PYTHON3=true
+
+# Use Py3 for subprocesses
+export PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS='["CPython>=3.6,<4"]'
+
+./pants "$@"

src/docs/export.md

@@ -101,7 +101,7 @@ The following is an abbreviated export file from a command in the pants repo:
     "python_setup": {
         "interpreters": {
             "CPython-2.7.10": {
-                "binary": "/Users/user/pants/build-support/pants_dev_deps.venv/bin/python2.7",
+                "binary": "/Users/user/pants/build-support/pants_dev_deps.py2.venv/bin/python2.7",
                 "chroot": "/Users/user/pants/.pants.d/python-setup/chroots/e8da2c200f36ca0a1b8a60c12590a59209250b1a"
             }
         },

src/docs/intellij.md

@@ -44,7 +44,7 @@ SDK".
 ![image](images/intellij-new-pythonsdk.png)
 
 This will be a "local" interpreter and you'll need to select the virtual
-environment bootstrapped above; it's in `build-support/pants_dev_deps.venv`.
+environment bootstrapped above; it's in `build-support/pants_dev_deps.py2.venv` if you want to use Python 2 or `build-support/pants_dev_deps.py3.venv` if you want to use Python 3 for the underlying Pants engine.
 
 ![image](images/intellij-select-venv.png)
 

src/python/pants/backend/python/subsystems/python_setup.py

@@ -26,7 +26,7 @@ class PythonSetup(Subsystem):
   @classmethod
   def register_options(cls, register):
     super(PythonSetup, cls).register_options(register)
-    register('--interpreter-constraints', advanced=True, default=['CPython>=2.7,<3'], type=list,
+    register('--interpreter-constraints', advanced=True, default=['CPython>=2.7,<3', 'CPython>=3.6,<4'], type=list,
              metavar='<requirement>',
              help="Constrain the selected Python interpreter.  Specify with requirement syntax, "
                   "e.g. 'CPython>=2.7,<3' (A CPython interpreter with version >=2.7 AND version <3)"

src/rust/engine/src/cffi_build.rs

@@ -109,6 +109,13 @@ fn main() -> Result<(), CffiBuildError> {
   //    (which is equivalent to `-ldylib=stdc++`).
   //  * Specifying `rustc-link-lib=stdc++`
   //    (which is equivalent to `rustc-link-lib=dylib=stdc++).
+
+  // NB: When built with Python 3, `native_engine.so` only works with a Python 3 interpreter.
+  // When built with Python 2, it works with both Python 2 and Python 3.
+  // So, we check to see if the under-the-hood interpreter has changed and rebuild the native engine
+  // when needed.
+  println!("cargo:rerun-if-env-changed=PANTS_USE_PYTHON3");
+
   if cfg!(target_os = "linux") {
     println!("cargo:rustc-link-lib=static=stdc++");
   }