Chore/python3

.github/main.workflow

@@ -0,0 +1,9 @@
+workflow "Run python formatter" {
+  on = "pull_request"
+  resolves = ["Run wool"]
+}
+
+action "Run wool" {
+  uses = "uc-cdis/wool@master"
+  secrets = ["GITHUB_TOKEN"]
+}

.travis.yml

@@ -6,7 +6,7 @@ python:
 
 matrix:
  allow_failures:
-   - python: "3.6"
+   - python: "2.7"
 
 sudo: false
 
@@ -15,9 +15,6 @@ cache: pip
 addons:
   postgresql: "9.4"
 
-env:
-  - REPOSITORY="uc-cdis/fence" PR_NUMBER="$TRAVIS_PULL_REQUEST"
-
 install:
   - pip uninstall -y six || true # travis installs wrong version
   - pip uninstall -y userdatamodel || true
@@ -27,7 +24,6 @@ install:
   - python setup.py install
   - psql -U postgres -c "create database fence_test_tmp"
   - if [[ $TRAVIS_PYTHON_VERSION != 3.6 ]]; then userdatamodel-init --db fence_test_tmp; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == 3.6 ]]; then pip install -e git+https://[email protected]/uc-cdis/wool.git#egg=wool; fi
   - pip list
 
 before_script:
@@ -42,4 +38,3 @@ script:
 after_script:
   - python-codacy-coverage -r coverage.xml
   - COVERALLS_REPO_TOKEN=$COVERALLS_TOKEN coveralls
-  - if [[ $TRAVIS_PYTHON_VERSION == 3.6 && $PR_NUMBER != false ]]; then wool; fi

Dockerfile

@@ -1,22 +1,31 @@
 # To run: docker run -d -v /path/to/fence-config.yaml:/var/www/fence/fence-config.yaml --name=fence -p 80:80 fence
 # To check running container: docker exec -it fence /bin/bash
 
-FROM quay.io/cdis/py27base:pybase2-1.0.2
+FROM quay.io/cdis/python-nginx:pybase3-1.0.0
 
-RUN mkdir /var/www/fence \
-    && chown www-data /var/www/fence
+ENV appname=fence
 
-COPY . /fence
+RUN apk update \
+    && apk add postgresql-libs postgresql-dev libffi-dev libressl-dev \
+    && apk add linux-headers musl-dev gcc \
+    && apk add curl bash git vim make
+
+COPY . /$appname
 COPY ./deployment/uwsgi/uwsgi.ini /etc/uwsgi/uwsgi.ini
+COPY ./deployment/uwsgi/wsgi.py /$appname/wsgi.py
+WORKDIR /$appname
 
-WORKDIR /fence
+RUN python -m pip install --upgrade pip \
+    && python -m pip install --upgrade setuptools \
+    && pip install -r requirements.txt
 
-RUN pip install --upgrade pip \
-  && python -m pip install -r requirements.txt
-RUN ln -s /fence/wsgi.py /var/www/fence/wsgi.py
-RUN COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >fence/version_data.py
-RUN VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>fence/version_data.py
-RUN python setup.py develop
+RUN mkdir -p /var/www/$appname \
+    && mkdir -p /var/www/.cache/Python-Eggs/ \
+    && mkdir /run/nginx/ \
+    && ln -sf /dev/stdout /var/log/nginx/access.log \
+    && ln -sf /dev/stderr /var/log/nginx/error.log \
+    && chown nginx -R /var/www/.cache/Python-Eggs/ \
+    && chown nginx /var/www/$appname
 
 RUN apk update && apk add openssh && apk add libmcrypt-dev
 
@@ -41,6 +50,10 @@ RUN (cd /tmp \
   && /bin/rm -rf /tmp/*)
 EXPOSE 80
 
-WORKDIR /var/www/fence
+RUN COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >$appname/version_data.py \
+    && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>$appname/version_data.py \
+    && python setup.py develop
+
+WORKDIR /var/www/$appname
 
-CMD ["sh","-c","bash /fence/dockerrun.bash && /dockerrun.sh"]
+CMD ["sh","-c","bash /fence/dockerrun.bash && /dockerrun.sh"]

Jenkinsfile

@@ -1,6 +1,6 @@
 #!groovy
 
-@Library('cdis-jenkins-lib@master') _
+@Library('cdis-jenkins-lib@chore/python3') _
 
 testPipeline { 
 }

README.md

@@ -101,6 +101,7 @@ At the moment, supported IDPs include:
   - InCommon
   - eduGAIN
 
+Note: the Shibboleth dockerfile image is at https://quay.io/repository/cdis/fence-shib and is NOT compatible with python 3/the latest fence.
 
 ## OIDC & OAuth2
 

bin/google-user-sa-validation.sh

@@ -21,4 +21,3 @@ while [ $? -eq 0 ]; do
     echo finish validation $(date)
     echo {\"last_run\": \"$(date)\"} >/google_job/status.json
 done
-

deployment/uwsgi/uwsgi.ini

@@ -2,9 +2,11 @@
 protocol = uwsgi
 socket = /var/run/gen3/uwsgi.sock
 buffer-size = 32768
+uid = nginx
+gid = nginx
+chown-socket = nginx:nginx
 chmod-socket = 666
 master = true
-processes = 2
 harakiri-verbose = true
 # No global HARAKIRI, using only user HARAKIRI, because export overwrites it
 # Cannot overwrite global HARAKIRI with user's: https://git.io/fjYuD
@@ -15,16 +17,14 @@ worker-reload-mercy = 45
 reload-mercy = 45
 mule-reload-mercy = 45
 disable-logging = true
-wsgi-file=/var/www/fence/wsgi.py
-plugins = python
+wsgi-file=/fence/wsgi.py
+plugins = python3
 vacuum = true
-uid = www-data
-gid = www-data
 pythonpath = /var/www/fence/
 pythonpath = /fence/
-pythonpath = /usr/local/lib/python2.7/dist-packages/
+pythonpath = /usr/local/lib/python3.6/site-packages/
 
-# Initialize application master to avoid race conditions when multiple workers
-# create the database.
-#lazy = true
-#lazy-apps = true
+# Initialize application in worker processes, not master. This prevents the
+# workers from all trying to open the same database connections at startup.
+# lazy = true
+# lazy-apps = true

deployment/uwsgi/wsgi.py

@@ -0,0 +1,4 @@
+from fence import app_init, app
+
+app_init(app)
+application = app

dev-requirements.txt

@@ -3,4 +3,4 @@ codacy-coverage
 coveralls
 mock==2.0.0
 moto==1.1.24
--e git+https://github.com/uc-cdis/cdisutils-test.git@0.2.5#egg=cdisutilstest-0.2.5
+-e git+https://github.com/uc-cdis/cdisutils-test.git@1.0.0#egg=cdisutilstest

fence/__init__.py

@@ -200,7 +200,7 @@ def app_config(
 
 def _setup_data_endpoint_and_boto(app):
     if "AWS_CREDENTIALS" in config and len(config["AWS_CREDENTIALS"]) > 0:
-        value = config["AWS_CREDENTIALS"].values()[0]
+        value = list(config["AWS_CREDENTIALS"].values())[0]
         app.boto = BotoManager(value, logger=logger)
         app.register_blueprint(fence.blueprints.data.blueprint, url_prefix="/data")
 
@@ -237,7 +237,7 @@ def _set_authlib_cfgs(app):
 
 
 def _setup_oidc_clients(app):
-    enabled_idp_ids = config["ENABLED_IDENTITY_PROVIDERS"]["providers"].keys()
+    enabled_idp_ids = list(config["ENABLED_IDENTITY_PROVIDERS"]["providers"].keys())
 
     # Add OIDC client for Google if configured.
     configured_google = (

fence/auth.py

@@ -1,7 +1,7 @@
 import flask
 from flask_sqlalchemy_session import current_session
 from functools import wraps
-import urllib
+import urllib.request, urllib.parse, urllib.error
 
 from authutils.errors import JWTError, JWTExpiredError
 from authutils.token.validate import (
@@ -99,16 +99,16 @@ def logout(next_url):
     provider_logout = None
     provider = flask.session.get("provider")
     if provider == IdentityProvider.itrust:
-        safe_url = urllib.quote_plus(next_url)
+        safe_url = urllib.parse.quote_plus(next_url)
         provider_logout = config["ITRUST_GLOBAL_LOGOUT"] + safe_url
     elif provider == IdentityProvider.fence:
         base = config["OPENID_CONNECT"]["fence"]["api_base_url"]
-        safe_url = urllib.quote_plus(next_url)
-        provider_logout = base + "/logout?" + urllib.urlencode({"next": safe_url})
+        safe_url = urllib.parse.quote_plus(next_url)
+        provider_logout = base + "/logout?" + urllib.parse.urlencode({"next": safe_url})
 
     flask.session.clear()
     redirect_response = flask.make_response(
-        flask.redirect(provider_logout or urllib.unquote(next_url))
+        flask.redirect(provider_logout or urllib.parse.unquote(next_url))
     )
     clear_cookies(redirect_response)
     return redirect_response

fence/blueprints/admin.py

@@ -24,15 +24,15 @@
 
 def debug_log(function):
     """Output debug information to the logger for a function call."""
-    argument_names = list(function.func_code.co_varnames)
+    argument_names = list(function.__code__.co_varnames)
 
     @functools.wraps(function)
     def write_log(*args, **kwargs):
         argument_values = (
             "{} = {}".format(arg, value)
-            for arg, value in zip(argument_names, args) + kwargs.items()
+            for arg, value in list(zip(argument_names, args)) + list(kwargs.items())
         )
-        msg = function.func_name + "\n\t" + "\n\t".join(argument_values)
+        msg = function.__name__ + "\n\t" + "\n\t".join(argument_values)
         logger.debug(msg)
         return function(*args, **kwargs)
 

fence/blueprints/data/indexd.py

@@ -1,6 +1,6 @@
 import re
 import time
-from urlparse import urlparse
+from urllib.parse import urlparse
 
 from cached_property import cached_property
 import cirrus
@@ -34,7 +34,7 @@
     get_google_app_creds,
 )
 from fence.utils import get_valid_expiration_from_request
-import multipart_upload
+from . import multipart_upload
 
 
 logger = get_logger(__name__)
@@ -825,7 +825,7 @@ def filter_auth_ids(action, list_auth_ids):
     elif action == "upload":
         checked_permission = "write-storage"
     authorized_dbgaps = []
-    for key, values in list_auth_ids.items():
+    for key, values in list(list_auth_ids.items()):
         if checked_permission in values:
             authorized_dbgaps.append(key)
     return authorized_dbgaps

fence/blueprints/google.py

@@ -1,6 +1,6 @@
 import os
 import json
-from urllib import unquote
+from urllib.parse import unquote
 from enum import Enum
 import time
 
@@ -458,16 +458,12 @@ def _update_service_account_permissions(self, sa):
 
         except CirrusNotFound as exc:
             return (
-                "Can not update the service accout {}. Detail {}".format(
-                    sa.email, exc.message
-                ),
+                "Can not update the service accout {}. Detail {}".format(sa.email, exc),
                 404,
             )
         except GoogleAPIError as exc:
             return (
-                "Can not update the service accout {}. Detail {}".format(
-                    sa.email, exc.message
-                ),
+                "Can not update the service accout {}. Detail {}".format(sa.email, exc),
                 400,
             )
         except Exception:
@@ -502,16 +498,12 @@ def _delete(self, id_):
             force_delete_service_account(service_account_email)
         except CirrusNotFound as exc:
             return (
-                "Can not remove the service accout {}. Detail {}".format(
-                    id_, exc.message
-                ),
+                "Can not remove the service accout {}. Detail {}".format(id_, exc),
                 404,
             )
         except GoogleAPIError as exc:
             return (
-                "Can not remove the service accout {}. Detail {}".format(
-                    id_, exc.message
-                ),
+                "Can not remove the service accout {}. Detail {}".format(id_, exc),
                 400,
             )
         except Exception:

fence/blueprints/login/__init__.py

@@ -83,7 +83,7 @@ def provider_info(idp_id):
             }
 
         try:
-            all_provider_info = [provider_info(idp_id) for idp_id in idps.keys()]
+            all_provider_info = [provider_info(idp_id) for idp_id in list(idps.keys())]
             default_provider_info = provider_info(default_idp)
         except KeyError as e:
             raise InternalError("identity providers misconfigured: {}".format(str(e)))

fence/blueprints/login/utils.py

@@ -1,4 +1,4 @@
-from urlparse import urlparse
+from urllib.parse import urlparse
 
 import flask
 

fence/blueprints/oauth2.py

@@ -230,7 +230,7 @@ def _get_auth_response_for_prompts(prompts, grant, user, client, scope):
 
         enabled_idps = config.get("OPENID_CONNECT", {})
         idp_names = []
-        for idp, info in enabled_idps.iteritems():
+        for idp, info in enabled_idps.items():
             # prefer name if its there, then just use the key for the provider
             idp_name = info.get("name") or idp.title()
             idp_names.append(idp_name)

fence/blueprints/storage_creds/__init__.py

@@ -72,7 +72,7 @@ def list_sources():
         services = set(
             [
                 info.get("backend")
-                for _, info in config["STORAGE_CREDENTIALS"].iteritems()
+                for _, info in config["STORAGE_CREDENTIALS"].items()
                 if info.get("backend")
             ]
         )

fence/config.py

@@ -1,6 +1,6 @@
 import os
 from yaml import safe_load as yaml_load
-import urlparse
+import urllib.parse
 
 import cirrus
 from gen3config import Config
@@ -45,7 +45,7 @@ def post_process(self):
             self.force_default_if_none(default, default_cfg=default_config)
 
         if "ROOT_URL" not in self._configs and "BASE_URL" in self._configs:
-            url = urlparse.urlparse(self._configs["BASE_URL"])
+            url = urllib.parse.urlparse(self._configs["BASE_URL"])
             self._configs["ROOT_URL"] = "{}://{}".format(url.scheme, url.netloc)
 
         # allow authlib traffic on http for development if enabled. By default

fence/error_handler.py

@@ -1,5 +1,5 @@
 import uuid
-from httplib import responses as http_responses
+from http.client import responses as http_responses
 import flask
 from flask import render_template
 from werkzeug.exceptions import HTTPException
@@ -49,12 +49,13 @@ def get_error_response(error):
 
 
 def get_error_details_and_status(error):
+    message = error.message if hasattr(error, "message") else str(error)
     if isinstance(error, APIError):
         if hasattr(error, "json") and error.json:
-            error.json["message"] = error.message
+            error.json["message"] = message
             error_response = error.json, error.code
         else:
-            error_response = {"message": error.message}, error.code
+            error_response = {"message": message}, error.code
     elif isinstance(error, OAuth2Error):
         error_response = {"message": error.description}, error.status_code
     elif isinstance(error, HTTPException):
@@ -69,7 +70,7 @@ def get_error_details_and_status(error):
             error_code = error.code
         elif hasattr(error, "status_code"):
             error_code = error.status_code
-        error_response = {"message": error.message}, error_code
+        error_response = {"message": message}, error_code
 
     return error_response