Allow Pants to run with Python 3 via ./pants3 script (#6959)

## Problem
We want to be able to use Pants using either Python 2 or Python 3 under-the-hood. (Note `--setup-python-interpreter-constraints` only constrains subprocesses, not the actual main Pants process.)

Further, the solution must still support Python 2. 

An ideal solution requires:

* Allow easily running using Python 3.
* Allow quickly falling back to Python 2 if Python 3 version is not working.
* Be transparent to the user which interpreter they are using.

## Solution
Add a new script `./pants3`, that functions identically to `./pants` beyond using Python 3 under-the-hood. 

The user can first attempt the command with `./pants3`, and if they're having an issue they can simply revert to `./pants` (along with filing an issue).

This solution has two coexisting `venv` folders, to avoid redownloading the virtual environment every time. 

### Other solutions considered
* Command line flag like `--py3`.
   * Too much typing for user every time. Is not easy to use Python 3.
   * Adds substantial boilerplate to `./pants` code.
* Environment variable
   * Not transparent which interpreter is being used.
   * Incorporated already into the `./pants3` solution. If the user does want to use an environment variable, they can as an alternative to `./pants3`.

## Once `./pants3` fully passes CI
`./pants3` will be renamed to `./pants`, which will always use Python 3, and `./pants` will be renamed to `./pants2`. 

We'll make this swap once `./pants3` passes all CI.

.travis.yml

@@ -43,7 +43,8 @@ native_engine_cache_config: &native_engine_cache_config
     timeout: 500
     directories:
       - ${HOME}/.cache/pants/rust/cargo
-      - build-support/pants_dev_deps.venv
+      - build-support/pants_dev_deps.py2.venv
+      - build-support/pants_dev_deps.py3.venv
       - src/rust/engine/target
 
 # Travis cache config for jobs that run a bootstrapped pants.pex.

build-support/pants_venv

@@ -12,7 +12,8 @@ REQUIREMENTS=(
 )
 
 function venv_dir() {
-  echo ${REPO_ROOT}/build-support/pants_dev_deps.venv
+  py_version="py2"; [ "${PANTS_USE_PYTHON3}" = true ] && py_version="py3"
+  echo ${REPO_ROOT}/build-support/pants_dev_deps.${py_version}.venv
 }
 
 function activate_venv() {
@@ -21,6 +22,15 @@ function activate_venv() {
 
 function create_venv() {
   rm -rf "$(venv_dir)"
+  # If `pants_dev_deps.venv` still exists, delete both its legacy folder and the cached
+  # Python interpreter folder, which contains problematic symlinks.
+  # Note we only perform these removals for the first time, to avoid continually deleting
+  # the cache when switching between using `pants_dev_deps.py2.venv` and `pants_dev_deps.py3.venv`.
+  legacy_venv_dir="${REPO_ROOT}/build-support/pants_dev_deps.venv"
+  if [ -d "${legacy_venv_dir}" ]; then
+    rm -rf "${legacy_venv_dir}"
+    rm -rf "${HOME}/.cache/pants/python_cache/interpreters"
+  fi
   "${REPO_ROOT}/build-support/virtualenv" "$(venv_dir)"
 }
 

build-support/python/clean.sh

@@ -3,5 +3,6 @@
 PANTS_BASE=$(dirname $0)/../..
 rm -rf ${HOME}/.pex
 rm -rf ${PANTS_BASE}/build-support/pants_dev_deps.venv
+rm -rf ${PANTS_BASE}/build-support/pants_dev_deps.py{2,3}.venv
 rm -rf ${PANTS_BASE}/.pants.d
 find ${PANTS_BASE} -name '*.pyc' | xargs rm -f

build-support/travis/travis.yml.mustache

@@ -39,7 +39,8 @@ native_engine_cache_config: &native_engine_cache_config
     timeout: 500
     directories:
       - ${HOME}/.cache/pants/rust/cargo
-      - build-support/pants_dev_deps.venv
+      - build-support/pants_dev_deps.py2.venv
+      - build-support/pants_dev_deps.py3.venv
       - src/rust/engine/target
 
 # Travis cache config for jobs that run a bootstrapped pants.pex.

build-support/virtualenv

@@ -14,10 +14,11 @@ fi
 source ${REPO_ROOT}/build-support/common.sh
 
 if [[ -z "${PY}" ]]; then
-  if which python2.7 >/dev/null; then
-    PY=`which python2.7`
+  python_bin_name="python2.7"; [ "${PANTS_USE_PYTHON3}" = true ] && python_bin_name="python3"
+  if which "${python_bin_name}" >/dev/null; then
+    PY=`which ${python_bin_name}`
   else
-    die 'No python2.7 interpreter found on the path.  Python will not work!'
+    die "No ${python_bin_name} interpreter found on the path. Python will not work!"
   fi
 fi
 

pants.ini

@@ -369,10 +369,11 @@ verify_commit: False
 
 
 [python-setup]
-# We only support pants running under 2.7 for now with 3.3+ support to be added later.
-# Any example or test targets that are meant to test interpreters outside pants own
-# acceptable set should specify an explicit compatibility constraint.
-interpreter_constraints: ["CPython>=2.7,<3"]
+# We will support Python 2 until release 1.16, at which we will drop Python 2 support and
+# require Python 3.6+ to run Pants.
+# Note this does not mean client code has to use these specified versions, only that
+# an appropriate interpreter must be discoverable.
+interpreter_constraints: ["CPython>=2.7,<3","CPython>=3.6,<4"]
 interpreter_cache_dir: %(pants_bootstrapdir)s/python_cache/interpreters
 resolver_cache_dir: %(pants_bootstrapdir)s/python_cache/requirements
 

pants3

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+# This bootstrap script invokes Pants using a Python 3 interpreter.
+
+# Use Py3 under-the-hood
+export PANTS_USE_PYTHON3=true
+
+# Use Py3 for subprocesses
+export PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS='["CPython>=3.6,<4"]'
+
+./pants "$@"

src/docs/export.md

@@ -101,7 +101,7 @@ The following is an abbreviated export file from a command in the pants repo:
     "python_setup": {
         "interpreters": {
             "CPython-2.7.10": {
-                "binary": "/Users/user/pants/build-support/pants_dev_deps.venv/bin/python2.7",
+                "binary": "/Users/user/pants/build-support/pants_dev_deps.py2.venv/bin/python2.7",
                 "chroot": "/Users/user/pants/.pants.d/python-setup/chroots/e8da2c200f36ca0a1b8a60c12590a59209250b1a"
             }
         },

src/docs/intellij.md

@@ -44,7 +44,7 @@ SDK".
 ![image](images/intellij-new-pythonsdk.png)
 
 This will be a "local" interpreter and you'll need to select the virtual
-environment bootstrapped above; it's in `build-support/pants_dev_deps.venv`.
+environment bootstrapped above; it's in `build-support/pants_dev_deps.py2.venv` if you want to use Python 2 or `build-support/pants_dev_deps.py3.venv` if you want to use Python 3 for the underlying Pants engine.
 
 ![image](images/intellij-select-venv.png)
 

src/python/pants/backend/python/subsystems/python_setup.py

@@ -26,7 +26,7 @@ class PythonSetup(Subsystem):
   @classmethod
   def register_options(cls, register):
     super(PythonSetup, cls).register_options(register)
-    register('--interpreter-constraints', advanced=True, default=['CPython>=2.7,<3'], type=list,
+    register('--interpreter-constraints', advanced=True, default=['CPython>=2.7,<3', 'CPython>=3.6,<4'], type=list,
              metavar='<requirement>',
              help="Constrain the selected Python interpreter.  Specify with requirement syntax, "
                   "e.g. 'CPython>=2.7,<3' (A CPython interpreter with version >=2.7 AND version <3)"

src/rust/engine/src/cffi_build.rs

@@ -109,6 +109,13 @@ fn main() -> Result<(), CffiBuildError> {
   //    (which is equivalent to `-ldylib=stdc++`).
   //  * Specifying `rustc-link-lib=stdc++`
   //    (which is equivalent to `rustc-link-lib=dylib=stdc++).
+
+  // NB: When built with Python 3, `native_engine.so` only works with a Python 3 interpreter.
+  // When built with Python 2, it works with both Python 2 and Python 3.
+  // So, we check to see if the under-the-hood interpreter has changed and rebuild the native engine
+  // when needed.
+  println!("cargo:rerun-if-env-changed=PANTS_USE_PYTHON3");
+
   if cfg!(target_os = "linux") {
     println!("cargo:rustc-link-lib=static=stdc++");
   }