diff --git a/.github/main.workflow b/.github/main.workflow new file mode 100644 index 000000000..fe47736a4 --- /dev/null +++ b/.github/main.workflow @@ -0,0 +1,9 @@ +workflow "Run python formatter" { + on = "pull_request" + resolves = ["Run wool"] +} + +action "Run wool" { + uses = "uc-cdis/wool@master" + secrets = ["GITHUB_TOKEN"] +} diff --git a/.travis.yml b/.travis.yml index 4426d8158..0db64bb13 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ python: matrix: allow_failures: - - python: "3.6" + - python: "2.7" sudo: false @@ -15,9 +15,6 @@ cache: pip addons: postgresql: "9.4" -env: - - REPOSITORY="uc-cdis/fence" PR_NUMBER="$TRAVIS_PULL_REQUEST" - install: - pip uninstall -y six || true # travis installs wrong version - pip uninstall -y userdatamodel || true @@ -27,7 +24,6 @@ install: - python setup.py install - psql -U postgres -c "create database fence_test_tmp" - if [[ $TRAVIS_PYTHON_VERSION != 3.6 ]]; then userdatamodel-init --db fence_test_tmp; fi - - if [[ $TRAVIS_PYTHON_VERSION == 3.6 ]]; then pip install -e git+https://git@github.com/uc-cdis/wool.git#egg=wool; fi - pip list before_script: @@ -42,4 +38,3 @@ script: after_script: - python-codacy-coverage -r coverage.xml - COVERALLS_REPO_TOKEN=$COVERALLS_TOKEN coveralls - - if [[ $TRAVIS_PYTHON_VERSION == 3.6 && $PR_NUMBER != false ]]; then wool; fi diff --git a/Dockerfile b/Dockerfile index 7432ede9b..a33318966 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,31 @@ # To run: docker run -d -v /path/to/fence-config.yaml:/var/www/fence/fence-config.yaml --name=fence -p 80:80 fence # To check running container: docker exec -it fence /bin/bash -FROM quay.io/cdis/py27base:pybase2-1.0.2 +FROM quay.io/cdis/python-nginx:pybase3-1.0.0 -RUN mkdir /var/www/fence \ - && chown www-data /var/www/fence +ENV appname=fence -COPY . /fence +RUN apk update \ + && apk add postgresql-libs postgresql-dev libffi-dev libressl-dev \ + && apk add linux-headers musl-dev gcc \ + && apk add curl bash git vim make + +COPY . /$appname COPY ./deployment/uwsgi/uwsgi.ini /etc/uwsgi/uwsgi.ini +COPY ./deployment/uwsgi/wsgi.py /$appname/wsgi.py +WORKDIR /$appname -WORKDIR /fence +RUN python -m pip install --upgrade pip \ + && python -m pip install --upgrade setuptools \ + && pip install -r requirements.txt -RUN pip install --upgrade pip \ - && python -m pip install -r requirements.txt -RUN ln -s /fence/wsgi.py /var/www/fence/wsgi.py -RUN COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >fence/version_data.py -RUN VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>fence/version_data.py -RUN python setup.py develop +RUN mkdir -p /var/www/$appname \ + && mkdir -p /var/www/.cache/Python-Eggs/ \ + && mkdir /run/nginx/ \ + && ln -sf /dev/stdout /var/log/nginx/access.log \ + && ln -sf /dev/stderr /var/log/nginx/error.log \ + && chown nginx -R /var/www/.cache/Python-Eggs/ \ + && chown nginx /var/www/$appname RUN apk update && apk add openssh && apk add libmcrypt-dev @@ -41,6 +50,10 @@ RUN (cd /tmp \ && /bin/rm -rf /tmp/*) EXPOSE 80 -WORKDIR /var/www/fence +RUN COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >$appname/version_data.py \ + && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>$appname/version_data.py \ + && python setup.py develop + +WORKDIR /var/www/$appname -CMD ["sh","-c","bash /fence/dockerrun.bash && /dockerrun.sh"] \ No newline at end of file +CMD ["sh","-c","bash /fence/dockerrun.bash && /dockerrun.sh"] diff --git a/Jenkinsfile b/Jenkinsfile index e7ed561a0..2fb0888c7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ #!groovy -@Library('cdis-jenkins-lib@master') _ +@Library('cdis-jenkins-lib@chore/python3') _ testPipeline { } diff --git a/README.md b/README.md index 79e008232..3a278b537 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ At the moment, supported IDPs include: - InCommon - eduGAIN +Note: the Shibboleth dockerfile image is at https://quay.io/repository/cdis/fence-shib and is NOT compatible with python 3/the latest fence. ## OIDC & OAuth2 diff --git a/bin/google-user-sa-validation.sh b/bin/google-user-sa-validation.sh index f882a2f93..8683a3707 100755 --- a/bin/google-user-sa-validation.sh +++ b/bin/google-user-sa-validation.sh @@ -21,4 +21,3 @@ while [ $? -eq 0 ]; do echo finish validation $(date) echo {\"last_run\": \"$(date)\"} >/google_job/status.json done - diff --git a/deployment/uwsgi/uwsgi.ini b/deployment/uwsgi/uwsgi.ini index 1a763b9b3..78bc72972 100644 --- a/deployment/uwsgi/uwsgi.ini +++ b/deployment/uwsgi/uwsgi.ini @@ -2,9 +2,11 @@ protocol = uwsgi socket = /var/run/gen3/uwsgi.sock buffer-size = 32768 +uid = nginx +gid = nginx +chown-socket = nginx:nginx chmod-socket = 666 master = true -processes = 2 harakiri-verbose = true # No global HARAKIRI, using only user HARAKIRI, because export overwrites it # Cannot overwrite global HARAKIRI with user's: https://git.io/fjYuD @@ -15,16 +17,14 @@ worker-reload-mercy = 45 reload-mercy = 45 mule-reload-mercy = 45 disable-logging = true -wsgi-file=/var/www/fence/wsgi.py -plugins = python +wsgi-file=/fence/wsgi.py +plugins = python3 vacuum = true -uid = www-data -gid = www-data pythonpath = /var/www/fence/ pythonpath = /fence/ -pythonpath = /usr/local/lib/python2.7/dist-packages/ +pythonpath = /usr/local/lib/python3.6/site-packages/ -# Initialize application master to avoid race conditions when multiple workers -# create the database. -#lazy = true -#lazy-apps = true +# Initialize application in worker processes, not master. This prevents the +# workers from all trying to open the same database connections at startup. +# lazy = true +# lazy-apps = true diff --git a/deployment/uwsgi/wsgi.py b/deployment/uwsgi/wsgi.py new file mode 100644 index 000000000..fc022f50f --- /dev/null +++ b/deployment/uwsgi/wsgi.py @@ -0,0 +1,4 @@ +from fence import app_init, app + +app_init(app) +application = app diff --git a/dev-requirements.txt b/dev-requirements.txt index b01a37a40..7091d886d 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,4 +3,4 @@ codacy-coverage coveralls mock==2.0.0 moto==1.1.24 --e git+https://github.com/uc-cdis/cdisutils-test.git@0.2.5#egg=cdisutilstest-0.2.5 +-e git+https://github.com/uc-cdis/cdisutils-test.git@1.0.0#egg=cdisutilstest diff --git a/fence/__init__.py b/fence/__init__.py index 051021b27..349ffd62e 100644 --- a/fence/__init__.py +++ b/fence/__init__.py @@ -200,7 +200,7 @@ def app_config( def _setup_data_endpoint_and_boto(app): if "AWS_CREDENTIALS" in config and len(config["AWS_CREDENTIALS"]) > 0: - value = config["AWS_CREDENTIALS"].values()[0] + value = list(config["AWS_CREDENTIALS"].values())[0] app.boto = BotoManager(value, logger=logger) app.register_blueprint(fence.blueprints.data.blueprint, url_prefix="/data") @@ -237,7 +237,7 @@ def _set_authlib_cfgs(app): def _setup_oidc_clients(app): - enabled_idp_ids = config["ENABLED_IDENTITY_PROVIDERS"]["providers"].keys() + enabled_idp_ids = list(config["ENABLED_IDENTITY_PROVIDERS"]["providers"].keys()) # Add OIDC client for Google if configured. configured_google = ( diff --git a/fence/auth.py b/fence/auth.py index f1942831d..c89ea2633 100644 --- a/fence/auth.py +++ b/fence/auth.py @@ -1,7 +1,7 @@ import flask from flask_sqlalchemy_session import current_session from functools import wraps -import urllib +import urllib.request, urllib.parse, urllib.error from authutils.errors import JWTError, JWTExpiredError from authutils.token.validate import ( @@ -99,16 +99,16 @@ def logout(next_url): provider_logout = None provider = flask.session.get("provider") if provider == IdentityProvider.itrust: - safe_url = urllib.quote_plus(next_url) + safe_url = urllib.parse.quote_plus(next_url) provider_logout = config["ITRUST_GLOBAL_LOGOUT"] + safe_url elif provider == IdentityProvider.fence: base = config["OPENID_CONNECT"]["fence"]["api_base_url"] - safe_url = urllib.quote_plus(next_url) - provider_logout = base + "/logout?" + urllib.urlencode({"next": safe_url}) + safe_url = urllib.parse.quote_plus(next_url) + provider_logout = base + "/logout?" + urllib.parse.urlencode({"next": safe_url}) flask.session.clear() redirect_response = flask.make_response( - flask.redirect(provider_logout or urllib.unquote(next_url)) + flask.redirect(provider_logout or urllib.parse.unquote(next_url)) ) clear_cookies(redirect_response) return redirect_response diff --git a/fence/blueprints/admin.py b/fence/blueprints/admin.py index d5c26f5e7..b44ca96d0 100644 --- a/fence/blueprints/admin.py +++ b/fence/blueprints/admin.py @@ -24,15 +24,15 @@ def debug_log(function): """Output debug information to the logger for a function call.""" - argument_names = list(function.func_code.co_varnames) + argument_names = list(function.__code__.co_varnames) @functools.wraps(function) def write_log(*args, **kwargs): argument_values = ( "{} = {}".format(arg, value) - for arg, value in zip(argument_names, args) + kwargs.items() + for arg, value in list(zip(argument_names, args)) + list(kwargs.items()) ) - msg = function.func_name + "\n\t" + "\n\t".join(argument_values) + msg = function.__name__ + "\n\t" + "\n\t".join(argument_values) logger.debug(msg) return function(*args, **kwargs) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index a87503f94..9e335a611 100644 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1,6 +1,6 @@ import re import time -from urlparse import urlparse +from urllib.parse import urlparse from cached_property import cached_property import cirrus @@ -34,7 +34,7 @@ get_google_app_creds, ) from fence.utils import get_valid_expiration_from_request -import multipart_upload +from . import multipart_upload logger = get_logger(__name__) @@ -825,7 +825,7 @@ def filter_auth_ids(action, list_auth_ids): elif action == "upload": checked_permission = "write-storage" authorized_dbgaps = [] - for key, values in list_auth_ids.items(): + for key, values in list(list_auth_ids.items()): if checked_permission in values: authorized_dbgaps.append(key) return authorized_dbgaps diff --git a/fence/blueprints/google.py b/fence/blueprints/google.py index 5e5d5057f..f977bb38a 100644 --- a/fence/blueprints/google.py +++ b/fence/blueprints/google.py @@ -1,6 +1,6 @@ import os import json -from urllib import unquote +from urllib.parse import unquote from enum import Enum import time @@ -458,16 +458,12 @@ def _update_service_account_permissions(self, sa): except CirrusNotFound as exc: return ( - "Can not update the service accout {}. Detail {}".format( - sa.email, exc.message - ), + "Can not update the service accout {}. Detail {}".format(sa.email, exc), 404, ) except GoogleAPIError as exc: return ( - "Can not update the service accout {}. Detail {}".format( - sa.email, exc.message - ), + "Can not update the service accout {}. Detail {}".format(sa.email, exc), 400, ) except Exception: @@ -502,16 +498,12 @@ def _delete(self, id_): force_delete_service_account(service_account_email) except CirrusNotFound as exc: return ( - "Can not remove the service accout {}. Detail {}".format( - id_, exc.message - ), + "Can not remove the service accout {}. Detail {}".format(id_, exc), 404, ) except GoogleAPIError as exc: return ( - "Can not remove the service accout {}. Detail {}".format( - id_, exc.message - ), + "Can not remove the service accout {}. Detail {}".format(id_, exc), 400, ) except Exception: diff --git a/fence/blueprints/login/__init__.py b/fence/blueprints/login/__init__.py index ac0484aa7..1977d989a 100644 --- a/fence/blueprints/login/__init__.py +++ b/fence/blueprints/login/__init__.py @@ -83,7 +83,7 @@ def provider_info(idp_id): } try: - all_provider_info = [provider_info(idp_id) for idp_id in idps.keys()] + all_provider_info = [provider_info(idp_id) for idp_id in list(idps.keys())] default_provider_info = provider_info(default_idp) except KeyError as e: raise InternalError("identity providers misconfigured: {}".format(str(e))) diff --git a/fence/blueprints/login/utils.py b/fence/blueprints/login/utils.py index b04d3f6ee..4b189977e 100644 --- a/fence/blueprints/login/utils.py +++ b/fence/blueprints/login/utils.py @@ -1,4 +1,4 @@ -from urlparse import urlparse +from urllib.parse import urlparse import flask diff --git a/fence/blueprints/oauth2.py b/fence/blueprints/oauth2.py index 1e148a16f..7830b9617 100644 --- a/fence/blueprints/oauth2.py +++ b/fence/blueprints/oauth2.py @@ -230,7 +230,7 @@ def _get_auth_response_for_prompts(prompts, grant, user, client, scope): enabled_idps = config.get("OPENID_CONNECT", {}) idp_names = [] - for idp, info in enabled_idps.iteritems(): + for idp, info in enabled_idps.items(): # prefer name if its there, then just use the key for the provider idp_name = info.get("name") or idp.title() idp_names.append(idp_name) diff --git a/fence/blueprints/storage_creds/__init__.py b/fence/blueprints/storage_creds/__init__.py index 62fa3e9d2..869cc9942 100644 --- a/fence/blueprints/storage_creds/__init__.py +++ b/fence/blueprints/storage_creds/__init__.py @@ -72,7 +72,7 @@ def list_sources(): services = set( [ info.get("backend") - for _, info in config["STORAGE_CREDENTIALS"].iteritems() + for _, info in config["STORAGE_CREDENTIALS"].items() if info.get("backend") ] ) diff --git a/fence/config.py b/fence/config.py index ff764fe0a..5c771d4ba 100644 --- a/fence/config.py +++ b/fence/config.py @@ -1,6 +1,6 @@ import os from yaml import safe_load as yaml_load -import urlparse +import urllib.parse import cirrus from gen3config import Config @@ -45,7 +45,7 @@ def post_process(self): self.force_default_if_none(default, default_cfg=default_config) if "ROOT_URL" not in self._configs and "BASE_URL" in self._configs: - url = urlparse.urlparse(self._configs["BASE_URL"]) + url = urllib.parse.urlparse(self._configs["BASE_URL"]) self._configs["ROOT_URL"] = "{}://{}".format(url.scheme, url.netloc) # allow authlib traffic on http for development if enabled. By default diff --git a/fence/error_handler.py b/fence/error_handler.py index 1e126a6a4..b287f3acb 100644 --- a/fence/error_handler.py +++ b/fence/error_handler.py @@ -1,5 +1,5 @@ import uuid -from httplib import responses as http_responses +from http.client import responses as http_responses import flask from flask import render_template from werkzeug.exceptions import HTTPException @@ -49,12 +49,13 @@ def get_error_response(error): def get_error_details_and_status(error): + message = error.message if hasattr(error, "message") else str(error) if isinstance(error, APIError): if hasattr(error, "json") and error.json: - error.json["message"] = error.message + error.json["message"] = message error_response = error.json, error.code else: - error_response = {"message": error.message}, error.code + error_response = {"message": message}, error.code elif isinstance(error, OAuth2Error): error_response = {"message": error.description}, error.status_code elif isinstance(error, HTTPException): @@ -69,7 +70,7 @@ def get_error_details_and_status(error): error_code = error.code elif hasattr(error, "status_code"): error_code = error.status_code - error_response = {"message": error.message}, error_code + error_response = {"message": message}, error_code return error_response diff --git a/fence/jwt/keys.py b/fence/jwt/keys.py index c192301a8..6e247eaed 100644 --- a/fence/jwt/keys.py +++ b/fence/jwt/keys.py @@ -109,9 +109,9 @@ def __init__(self, kid, public_key, private_key): # Raise an error if either key does not match our expectations that the # private key should be private and the public key should not be # private. - if "PRIVATE KEY" not in private_key: + if "PRIVATE KEY" not in str(private_key): raise ValueError("received private key that was not an RSA private key") - if "PRIVATE KEY" in public_key: + if "PRIVATE KEY" in str(public_key): raise ValueError("received public key that was actually an RSA private key") self.kid = kid @@ -181,8 +181,13 @@ def public_key_to_jwk(self): Return: dict: JWK representation of the public key """ - n, e = _rsa_public_numbers(self.public_key) jwk_dict = jwk.construct(self.public_key, algorithm="RS256").to_dict() + for k in jwk_dict: # convert byte values to string + try: + jwk_dict[k] = jwk_dict[k].decode("utf-8") + except AttributeError: + # there is no need to decode values that are already strings + pass jwk_dict.update({"use": "sig", "key_ops": "verify", "kid": self.kid}) return jwk_dict @@ -200,7 +205,9 @@ def _rsa_public_numbers(public_key_data): Return: Tuple[int, int]: the public key modulus ``n`` and exponent ``e`` """ - key = serialization.load_pem_public_key(public_key_data, default_backend()) + key = serialization.load_pem_public_key( + bytes(public_key_data, "utf-8"), default_backend() + ) numbers = key.public_numbers() return (numbers.n, numbers.e) diff --git a/fence/jwt/validate.py b/fence/jwt/validate.py index 0f6e3d7e9..bf64e0c81 100644 --- a/fence/jwt/validate.py +++ b/fence/jwt/validate.py @@ -89,7 +89,7 @@ def validate_jwt( try: token_iss = jwt.decode(encoded_token, verify=False).get("iss") except jwt.InvalidTokenError as e: - raise JWTError(e.message) + raise JWTError(e) attempt_refresh = attempt_refresh and (token_iss != iss) public_key = authutils.token.keys.get_public_key_for_token( encoded_token, attempt_refresh=attempt_refresh diff --git a/fence/models.py b/fence/models.py index deaf9f8d1..c147123bc 100644 --- a/fence/models.py +++ b/fence/models.py @@ -201,7 +201,7 @@ def check_client_type(self, client_type): def check_client_secret(self, client_secret): check_hash = bcrypt.hashpw( client_secret.encode("utf-8"), self.client_secret.encode("utf-8") - ) + ).decode("utf-8") return check_hash == self.client_secret def check_requested_scopes(self, scopes): @@ -1006,7 +1006,7 @@ def _update_for_authlib(driver, md): add_client_col = lambda col: add_column_if_not_exist( Client.__tablename__, column=col, driver=driver, metadata=md ) - map(add_client_col, CLIENT_COLUMNS_TO_ADD) + list(map(add_client_col, CLIENT_COLUMNS_TO_ADD)) CODE_COLUMNS_TO_ADD = [Column("response_type", Text, default="")] with driver.session as session: @@ -1023,7 +1023,7 @@ def _update_for_authlib(driver, md): add_code_col = lambda col: add_column_if_not_exist( AuthorizationCode.__tablename__, column=col, driver=driver, metadata=md ) - map(add_code_col, CODE_COLUMNS_TO_ADD) + list(map(add_code_col, CODE_COLUMNS_TO_ADD)) with driver.session as session: session.execute("ALTER TABLE client ALTER COLUMN client_secret DROP NOT NULL") session.commit() diff --git a/fence/oidc/endpoints.py b/fence/oidc/endpoints.py index 1edcb10d0..1b5b0fe6c 100644 --- a/fence/oidc/endpoints.py +++ b/fence/oidc/endpoints.py @@ -56,7 +56,9 @@ def validate_authenticate_client(self): # authorization header to check against stored hash. hashed = client.client_secret if ( - bcrypt.hashpw(client_secret.encode("utf-8"), hashed.encode("utf-8")) + bcrypt.hashpw(client_secret.encode("utf-8"), hashed.encode("utf-8")).decode( + "utf-8" + ) != hashed ): logger.debug("client secret hash does not match stored secret hash") diff --git a/fence/oidc/grants/authorization_code_grant.py b/fence/oidc/grants/authorization_code_grant.py index 036efec67..adb5f0e6b 100644 --- a/fence/oidc/grants/authorization_code_grant.py +++ b/fence/oidc/grants/authorization_code_grant.py @@ -135,7 +135,9 @@ def authenticate_client(self): # Client secrets are stored as hash. hashed = client.client_secret if ( - bcrypt.hashpw(client_secret.encode("utf-8"), hashed.encode("utf-8")) + bcrypt.hashpw( + client_secret.encode("utf-8"), hashed.encode("utf-8") + ).decode("utf-8") != hashed ): raise InvalidClientError(uri=self.uri) diff --git a/fence/oidc/grants/refresh_token_grant.py b/fence/oidc/grants/refresh_token_grant.py index 6b160e820..026002b87 100644 --- a/fence/oidc/grants/refresh_token_grant.py +++ b/fence/oidc/grants/refresh_token_grant.py @@ -1,4 +1,3 @@ -import bcrypt import flask from authlib.oauth2.rfc6749.errors import ( diff --git a/fence/rbac/client.py b/fence/rbac/client.py index eb18b7e44..8724730c0 100644 --- a/fence/rbac/client.py +++ b/fence/rbac/client.py @@ -5,7 +5,7 @@ from functools import wraps import json -import urllib +import urllib.request, urllib.parse, urllib.error import backoff from cdislogging import get_logger @@ -428,7 +428,7 @@ def grant_user_policy(self, username, policy_id): @_arborist_retry() def revoke_all_policies_for_user(self, username): - url = self._user_url + "/{}/policy".format(urllib.quote(username)) + url = self._user_url + "/{}/policy".format(urllib.parse.quote(username)) response = requests.delete(url) data = _request_get_json(response) if response.status_code != 204: diff --git a/fence/resources/admin/admin_groups.py b/fence/resources/admin/admin_groups.py index 5d0cb118c..7640daea3 100644 --- a/fence/resources/admin/admin_groups.py +++ b/fence/resources/admin/admin_groups.py @@ -76,7 +76,7 @@ def update_group_users_projects(current_session, group, project, users): proj = pj.get_project(current_session, project) for user in users: try: - user_projects = user.project_access.keys() + user_projects = list(user.project_access.keys()) if project not in user_projects: project_info = {"auth_id": proj.auth_id, "privilege": ["read"]} au.connect_user_to_project( diff --git a/fence/resources/aws/boto_manager.py b/fence/resources/aws/boto_manager.py index f4d0a98bf..3c96d8cbf 100644 --- a/fence/resources/aws/boto_manager.py +++ b/fence/resources/aws/boto_manager.py @@ -50,11 +50,11 @@ def delete_data_file(self, bucket, guid): ) except (KeyError, Boto3Error) as e: self.logger.exception(e) - raise InternalError("Failed to delete file: {}".format(e.message)) + raise InternalError("Failed to delete file: {}".format(str(e))) def assume_role(self, role_arn, duration_seconds, config=None): try: - if config and config.has_key("aws_access_key_id"): + if config and "aws_access_key_id" in config: self.sts_client = client("sts", **config) session_name_postfix = uuid.uuid4() return self.sts_client.assume_role( @@ -64,10 +64,10 @@ def assume_role(self, role_arn, duration_seconds, config=None): ) except Boto3Error as ex: self.logger.exception(ex) - raise InternalError("Fail to assume role: {}".format(ex.message)) + raise InternalError("Fail to assume role: {}".format(ex)) except Exception as ex: self.logger.exception(ex) - raise UnavailableError("Fail to reach AWS: {}".format(ex.message)) + raise UnavailableError("Fail to reach AWS: {}".format(ex)) def presigned_url(self, bucket, key, expires, config, method="get_object"): """ @@ -80,7 +80,7 @@ def presigned_url(self, bucket, key, expires, config, method="get_object"): """ if method not in ["get_object", "put_object"]: raise UserError("method {} not allowed".format(method)) - if config.has_key("aws_access_key_id"): + if "aws_access_key_id" in config: self.s3_client = client("s3", **config) expires = int(expires) or self.URL_EXPIRATION_DEFAULT expires = min(expires, self.URL_EXPIRATION_MAX) @@ -93,16 +93,16 @@ def presigned_url(self, bucket, key, expires, config, method="get_object"): def get_bucket_region(self, bucket, config): try: - if config.has_key("aws_access_key_id"): + if "aws_access_key_id" in config: self.s3_client = client("s3", **config) response = self.s3_client.get_bucket_location(Bucket=bucket) region = response.get("LocationConstraint") except Boto3Error as ex: self.logger.exception(ex) - raise InternalError("Fail to get bucket region: {}".format(ex.message)) + raise InternalError("Fail to get bucket region: {}".format(ex)) except Exception as ex: self.logger.exception(ex) - raise UnavailableError("Fail to reach AWS: {}".format(ex.message)) + raise UnavailableError("Fail to reach AWS: {}".format(ex)) if region is None: return "us-east-1" return region @@ -122,7 +122,7 @@ def get_all_groups(self, list_group_name): groups[group_name] = self.create_user_group(group_name) except Exception as ex: self.logger.exception(ex) - raise UserError("Fail to create list of groups: {}".format(ex.message)) + raise UserError("Fail to create list of groups: {}".format(ex)) return groups def add_user_to_group(self, groups, username): @@ -133,13 +133,13 @@ def add_user_to_group(self, groups, username): :return: """ try: - for group in groups.values(): + for group in list(groups.values()): self.iam.add_user_to_group( GroupName=group["GroupName"], UserName=username ) except Exception as ex: self.logger.exception(ex) - raise UserError("Fail to add user to group: {}".format(ex.message)) + raise UserError("Fail to add user to group: {}".format(ex)) def get_user_group(self, group_names): try: @@ -150,9 +150,7 @@ def get_user_group(self, group_names): res[group["GroupName"]] = group except Exception as ex: self.logger.exception(ex) - raise UserError( - "Fail to get list of groups {}: {}".format(group_names, ex.message) - ) + raise UserError("Fail to get list of groups {}: {}".format(group_names, ex)) return res def create_user_group(self, group_name, path=None): @@ -163,9 +161,7 @@ def create_user_group(self, group_name, path=None): ) except Exception as ex: self.logger.exception(ex) - raise UserError( - "Fail to create group {}: {}".format(group_name, ex.message) - ) + raise UserError("Fail to create group {}: {}".format(group_name, ex)) return group def __get_policy_document_by_group_name__(self, group_name): @@ -192,7 +188,7 @@ def __create_policy__( """ try: aws_kwargs = dict(Path=path, Description=description) - aws_kwargs = {k: v for k, v in aws_kwargs.items() if v is not None} + aws_kwargs = {k: v for k, v in list(aws_kwargs.items()) if v is not None} policy = self.iam.create_policy( PolicyName=policy_name, PolicyDocument=policy_document, **aws_kwargs ) @@ -201,5 +197,5 @@ def __create_policy__( ) except Exception as ex: self.logger.exception(ex) - raise UserError("Fail to create policy: {}".format(ex.message)) + raise UserError("Fail to create policy: {}".format(ex)) return policy diff --git a/fence/resources/google/access_utils.py b/fence/resources/google/access_utils.py index dc643b66e..6e4e6ba05 100644 --- a/fence/resources/google/access_utils.py +++ b/fence/resources/google/access_utils.py @@ -4,7 +4,7 @@ """ import time import flask -from urllib import unquote +from urllib.parse import unquote from cirrus.google_cloud.iam import GooglePolicyMember diff --git a/fence/resources/google/utils.py b/fence/resources/google/utils.py index d9725424c..e95cfeaad 100644 --- a/fence/resources/google/utils.py +++ b/fence/resources/google/utils.py @@ -72,7 +72,7 @@ def get_or_create_primary_service_account_key( if user_service_account_key: fernet_key = Fernet(str(config["ENCRYPTION_KEY"])) private_key_bytes = fernet_key.decrypt( - str(user_service_account_key.private_key) + bytes(user_service_account_key.private_key, "utf-8") ) sa_private_key = json.loads(private_key_bytes.decode("utf-8")) else: @@ -140,7 +140,7 @@ def create_primary_service_account_key(user_id, username, proxy_group_id, expire fernet_key = Fernet(str(config["ENCRYPTION_KEY"])) private_key_bytes = json.dumps(sa_private_key).encode("utf-8") - private_key = fernet_key.encrypt(private_key_bytes) + private_key = fernet_key.encrypt(private_key_bytes).decode("utf-8") expires = expires or ( int(time.time()) diff --git a/fence/resources/google/validity.py b/fence/resources/google/validity.py index 2ce34bdff..ea372401e 100644 --- a/fence/resources/google/validity.py +++ b/fence/resources/google/validity.py @@ -73,7 +73,7 @@ def __contains__(self, key): return key in self._info def __iter__(self): - for key, value in self._info.iteritems(): + for key, value in self._info.items(): yield key, value def __getitem__(self, key): @@ -88,9 +88,6 @@ def __len__(self): def __bool__(self): return self._valid - def __nonzero__(self): - return self._valid - def __repr__(self): return str(self._info) @@ -347,7 +344,7 @@ def check_validity(self, early_return=True, db=None): self.set("members_exist_in_fence", False) logger.warning( "INVALID user(s) do not exist in fence and thus, " - "we cannot determine their authZ info: {}.".format(e.message) + "we cannot determine their authZ info: {}.".format(e) ) if early_return: return diff --git a/fence/resources/openid/google_oauth2.py b/fence/resources/openid/google_oauth2.py index c5a1a61d1..506c96a6c 100644 --- a/fence/resources/openid/google_oauth2.py +++ b/fence/resources/openid/google_oauth2.py @@ -1,4 +1,4 @@ -from idp_oauth2 import Oauth2ClientBase +from .idp_oauth2 import Oauth2ClientBase class GoogleOauth2Client(Oauth2ClientBase): diff --git a/fence/resources/openid/microsoft_oauth2.py b/fence/resources/openid/microsoft_oauth2.py index b1d35b0be..dc8749a74 100644 --- a/fence/resources/openid/microsoft_oauth2.py +++ b/fence/resources/openid/microsoft_oauth2.py @@ -1,4 +1,4 @@ -from idp_oauth2 import Oauth2ClientBase +from .idp_oauth2 import Oauth2ClientBase class MicrosoftOauth2Client(Oauth2ClientBase): diff --git a/fence/resources/openid/orcid_oauth2.py b/fence/resources/openid/orcid_oauth2.py index fd3af8cd2..cd7190684 100644 --- a/fence/resources/openid/orcid_oauth2.py +++ b/fence/resources/openid/orcid_oauth2.py @@ -1,4 +1,4 @@ -from idp_oauth2 import Oauth2ClientBase +from .idp_oauth2 import Oauth2ClientBase class OrcidOauth2Client(Oauth2ClientBase): diff --git a/fence/resources/storage/__init__.py b/fence/resources/storage/__init__.py index 92c634704..8b24f4b79 100644 --- a/fence/resources/storage/__init__.py +++ b/fence/resources/storage/__init__.py @@ -50,7 +50,7 @@ class StorageManager(object): def __init__(self, credentials, logger): self.logger = logger self.clients = {} - for provider, config in credentials.iteritems(): + for provider, config in credentials.items(): if "backend" not in config: self.logger.error( "Storage provider {} is not configured with backend".format( @@ -69,12 +69,12 @@ def check_auth(self, provider, user): check if the user should be authorized to storage resources """ storage_access = any( - ["read-storage" in item for item in user.project_access.values()] + ["read-storage" in item for item in list(user.project_access.values())] ) backend_access = any( [ sa.provider.name == provider - for p in user.projects.values() + for p in list(user.projects.values()) for sa in p.storage_access ] ) diff --git a/fence/resources/storage/cdis_jwt.py b/fence/resources/storage/cdis_jwt.py index e8acb4ab4..ca3e70ac8 100644 --- a/fence/resources/storage/cdis_jwt.py +++ b/fence/resources/storage/cdis_jwt.py @@ -14,7 +14,7 @@ def create_access_token(user, keypair, api_key, expires_in, scopes): if not set(claims["aud"]).issuperset(scopes): raise JWTError("cannot issue access token with scope beyond refresh token") except Exception as e: - return flask.jsonify({"errors": e.message}) + return flask.jsonify({"errors": str(e)}) return token.generate_signed_access_token( keypair.kid, keypair.private_key, user, expires_in, scopes ).token @@ -57,7 +57,7 @@ def create_user_access_token(keypair, api_key, expires_in): scopes = claims["aud"] user = get_user_from_claims(claims) except Exception as e: - raise Unauthorized(e.message) + raise Unauthorized(str(e)) return token.generate_signed_access_token( keypair.kid, keypair.private_key, user, expires_in, scopes ).token diff --git a/fence/resources/user/__init__.py b/fence/resources/user/__init__.py index 0f19c8b52..f5697ffcc 100644 --- a/fence/resources/user/__init__.py +++ b/fence/resources/user/__init__.py @@ -124,7 +124,7 @@ def get_user_info(current_session, username): def _get_optional_userinfo(user, claims): info = {} - for claim, claim_request in claims.iteritems(): + for claim in claims: if claim == "linked_google_account": google_email = get_linked_google_account_email(user.id) info["linked_google_account"] = google_email diff --git a/fence/resources/userdatamodel/__init__.py b/fence/resources/userdatamodel/__init__.py index 6d7c317a1..63e205b3c 100644 --- a/fence/resources/userdatamodel/__init__.py +++ b/fence/resources/userdatamodel/__init__.py @@ -4,7 +4,7 @@ database. """ -from userdatamodel_project import * -from userdatamodel_user import * -from userdatamodel_group import * -from userdatamodel_provider import * +from .userdatamodel_project import * +from .userdatamodel_user import * +from .userdatamodel_group import * +from .userdatamodel_provider import * diff --git a/fence/scripting/fence_create.py b/fence/scripting/fence_create.py index 47d7ceb6b..3f02f8072 100644 --- a/fence/scripting/fence_create.py +++ b/fence/scripting/fence_create.py @@ -61,7 +61,7 @@ def list_client_action(db): for row in s.query(Client).all(): pprint.pprint(row.__dict__) except Exception as e: - print(e.message) + print(str(e)) def modify_client_action( @@ -114,7 +114,7 @@ def create_client_action( ) ) except Exception as e: - print(e.message) + print(str(e)) def delete_client_action(DB, client_name): @@ -146,7 +146,7 @@ def delete_client_action(DB, client_name): print("Client {} deleted".format(client_name)) except Exception as e: - print(e.message) + print(str(e)) def _remove_client_service_accounts(db_session, client): @@ -260,7 +260,7 @@ def create_sample_data(DB, yaml_input): def create_group(s, data): - for group_name, fields in data["groups"].iteritems(): + for group_name, fields in data["groups"].items(): projects = fields.get("projects", []) group = s.query(Group).filter(Group.name == group_name).first() if not group: @@ -373,7 +373,7 @@ def grant_project_to_group_or_user(s, project_data, group=None, user=None): def create_cloud_providers(s, data): cloud_data = data.get("cloud_providers", []) - for name, fields in cloud_data.iteritems(): + for name, fields in cloud_data.items(): cloud_provider = ( s.query(CloudProvider).filter(CloudProvider.name == name).first() ) @@ -389,7 +389,7 @@ def create_cloud_providers(s, data): def create_users_with_group(DB, s, data): providers = {} data_groups = data["groups"] - for username, data in data["users"].iteritems(): + for username, data in data["users"].items(): is_existing_user = True user = query_for_user(session=s, username=username) @@ -606,7 +606,7 @@ def delete_expired_service_accounts(DB): except Exception as e: print( "ERROR: Could not delete service account {}. Details: {}".format( - record.service_account.email, e.message + record.service_account.email, e ) ) @@ -635,7 +635,7 @@ def verify_bucket_access_group(DB): try: members = manager.get_group_members(access_group.email) except GoogleAuthError as e: - print("ERROR: Authentication error!!!. Detail {}".format(e.message)) + print("ERROR: Authentication error!!!. Detail {}".format(e)) return except Exception as e: print( diff --git a/fence/scripting/google_monitor.py b/fence/scripting/google_monitor.py index d51f63784..4e988c382 100644 --- a/fence/scripting/google_monitor.py +++ b/fence/scripting/google_monitor.py @@ -61,7 +61,7 @@ def validation_check(db): registered_service_accounts ) - for google_project_id, sa_emails in project_service_account_mapping.iteritems(): + for google_project_id, sa_emails in project_service_account_mapping.items(): email_required = False invalid_registered_service_account_reasons = {} invalid_project_reasons = {} @@ -476,7 +476,7 @@ def _send_emails_informing_service_account_removal( text = config["REMOVE_SERVICE_ACCOUNT_EMAIL_NOTIFICATION"]["content"] content = text.format(project_id) - for email, removal_reasons in invalid_service_account_reasons.iteritems(): + for email, removal_reasons in invalid_service_account_reasons.items(): if removal_reasons: content += "\n\t - Service account {} was removed from Google Project {}.".format( email, project_id @@ -499,12 +499,12 @@ def _send_emails_informing_service_account_removal( content += "\n\t\t - {}".format(removal_reason) if access_errors: - for project, removal_reasons in access_errors.iteritems(): + for project, removal_reasons in access_errors.items(): for reason in removal_reasons: content += "\n\t\t - {}".format(reason) if non_reg_sa_errors: - for sa_email, removal_reasons in non_reg_sa_errors.iteritems(): + for sa_email, removal_reasons in non_reg_sa_errors.items(): content += "\n\t\t - Google Project Service Account {} determined invalid.".format( sa_email ) @@ -647,7 +647,7 @@ def email_users_without_access( if member.member_type == GooglePolicyMember.USER: users.append(member.email_id) - for user, projects in users_without_access.iteritems(): + for user, projects in users_without_access.items(): logger.info( "{} does not have access to the following datasets: {}.".format( user, ",".join(projects) diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py index 9ef18c070..c5d2765da 100644 --- a/fence/sync/sync_users.py +++ b/fence/sync/sync_users.py @@ -5,7 +5,7 @@ import glob import os import re -from StringIO import StringIO +from io import StringIO import subprocess as sp import tempfile import shutil @@ -162,7 +162,7 @@ def from_file(cls, filepath, encrypted=True, key=None, logger=None): project_to_resource = dict() users = data.get("users", {}) - for username, details in users.iteritems(): + for username, details in users.items(): # users should occur only once each; skip if already processed if username in projects: if logger: @@ -205,7 +205,7 @@ def from_file(cls, filepath, encrypted=True, key=None, logger=None): # resources should be the resource tree to construct in arborist user_rbac = dict() - for username, details in users.iteritems(): + for username, details in users.items(): # users should occur only once each; skip if already processed if username in user_rbac: msg = "invalid yaml file: user `{}` occurs multiple times".format( @@ -425,7 +425,7 @@ def _parse_csv(self, file_dict, sess, encrypted=True): """ user_projects = dict() user_info = dict() - for filepath, privileges in file_dict.iteritems(): + for filepath, privileges in file_dict.items(): self.logger.info("Reading file {}".format(filepath)) if os.stat(filepath).st_size == 0: continue @@ -554,11 +554,11 @@ def sync_two_phsids_dict(phsids1, phsids2): For the other cases, just simple addition """ - for user, projects1 in phsids1.iteritems(): + for user, projects1 in phsids1.items(): if not phsids2.get(user): phsids2[user] = projects1 else: - for phsid1, privilege1 in projects1.iteritems(): + for phsid1, privilege1 in projects1.items(): if phsid1 not in phsids2[user]: phsids2[user][phsid1] = set() phsids2[user][phsid1].update(privilege1) @@ -599,13 +599,13 @@ def sync_to_db_and_storage_backend(self, user_project, user_info, sess): # db stores case-sensitively, but we need to query case-insensitively user_project_lowercase = {} syncing_user_project_list = set() - for username, projects in user_project.iteritems(): + for username, projects in user_project.items(): user_project_lowercase[username.lower()] = projects - for project, _ in projects.iteritems(): + for project, _ in projects.items(): syncing_user_project_list.add((username.lower(), project)) user_info_lowercase = { - username.lower(): info for username, info in user_info.iteritems() + username.lower(): info for username, info in user_info.items() } to_delete = set.difference(cur_db_user_project_list, syncing_user_project_list) @@ -788,7 +788,7 @@ def _upsert_userinfo(self, sess, user_info): u.tags.remove(tag) # sync - for k, v in user_info[username]["tags"].iteritems(): + for k, v in user_info[username]["tags"].items(): found = False for tag in u.tags: if tag.key == k: @@ -863,15 +863,15 @@ def _init_projects(self, user_project, sess): initialize projects """ if self.project_mapping: - for projects in self.project_mapping.values(): + for projects in list(self.project_mapping.values()): for p in projects: self.logger.debug( "creating Project with info from project_mapping: {}".format(p) ) project = self._get_or_create(sess, Project, **p) self._projects[p["auth_id"]] = project - for _, projects in user_project.iteritems(): - for auth_id in projects.keys(): + for _, projects in user_project.items(): + for auth_id in list(projects.keys()): project = sess.query(Project).filter(Project.auth_id == auth_id).first() if not project: data = {"name": auth_id, "auth_id": auth_id} @@ -925,7 +925,7 @@ def _sync(self, sess): self.logger.info("dbgap files: {}".format(dbgap_file_list)) permissions = [{"read-storage"} for _ in dbgap_file_list] user_projects, user_info = self._parse_csv( - dict(zip(dbgap_file_list, permissions)), encrypted=True, sess=sess + dict(list(zip(dbgap_file_list, permissions))), encrypted=True, sess=sess ) try: shutil.rmtree(tmpdir) @@ -942,7 +942,9 @@ def _sync(self, sess): permissions = [{"read-storage"} for _ in local_csv_file_list] user_projects_csv, user_info_csv = self._parse_csv( - dict(zip(local_csv_file_list, permissions)), encrypted=False, sess=sess + dict(list(zip(local_csv_file_list, permissions))), + encrypted=False, + sess=sess, ) try: @@ -955,11 +957,11 @@ def _sync(self, sess): return user_projects_csv = { - key.lower(): value for key, value in user_projects_csv.iteritems() + key.lower(): value for key, value in user_projects_csv.items() } - user_projects = {key.lower(): value for key, value in user_projects.iteritems()} + user_projects = {key.lower(): value for key, value in user_projects.items()} user_yaml.projects = { - key.lower(): value for key, value in user_yaml.projects.iteritems() + key.lower(): value for key, value in user_yaml.projects.items() } self.sync_two_phsids_dict(user_projects_csv, user_projects) @@ -1133,14 +1135,14 @@ def _update_authz_in_arborist(self, session, user_projects, user_yaml=None): # update the project info with `projects` specified in user.yaml self.sync_two_phsids_dict(user_yaml.user_rbac, user_projects) - for username, user_project_info in user_projects.iteritems(): + for username, user_project_info in user_projects.items(): self.logger.info("processing user `{}`".format(username)) user = query_for_user(session=session, username=username) self.arborist_client.create_user_if_not_exist(user.username) self.arborist_client.revoke_all_policies_for_user(user.username) - for project, permissions in user_project_info.iteritems(): + for project, permissions in user_project_info.items(): # check if this is a dbgap project, if it is, we need to get the right # resource path, otherwise just use given project as path @@ -1204,7 +1206,7 @@ def _update_authz_in_arborist(self, session, user_projects, user_yaml=None): for policy in user_yaml.policies.get(user.username, []): self.arborist_client.grant_user_policy(user.username, policy) - for client_name, client_details in user_yaml.clients.iteritems(): + for client_name, client_details in user_yaml.clients.items(): client_policies = client_details.get("policies", []) client = session.query(Client).filter_by(name=client_name).first() # update existing clients, do not create new ones diff --git a/fence/sync/utils.py b/fence/sync/utils.py index a1b4f8b6b..46100d3df 100644 --- a/fence/sync/utils.py +++ b/fence/sync/utils.py @@ -1,3 +1,6 @@ +from functools import reduce + + def combine_provided_and_dbgap_resources(useryaml_resources, arborist_paths): """ Combine provided user.yaml resources loaded into python list of dictionaries @@ -133,12 +136,12 @@ def insert_segment(current, segment): # for future reference on what this is doing, an example: # In [1]: xs = [{"name": "a"}, {"name": "b"}, {"name": "c"}] - # In [2]: list(map(lambda x: x["name"] == "b", xs)) + # In [2]: list([c["name"] == "b" for c in xs]) # Out[2]: [False, True, False] - # In [3]: list(map(lambda x: x["name"] == "b", xs)).index(True) + # In [3]: list([c["name"] == "b" for c in xs]).index(True) # Out[3]: 1 - i = list(map(lambda c: c["name"] == segment, current)).index(True) + i = list([c["name"] == segment for c in current]).index(True) if "subresources" not in current[i]: current[i]["subresources"] = [] diff --git a/fence/utils.py b/fence/utils.py index 24976dc5c..962cc5de2 100644 --- a/fence/utils.py +++ b/fence/utils.py @@ -6,8 +6,8 @@ import re import string import requests -from urllib import urlencode -from urlparse import parse_qs, urlsplit, urlunsplit +from urllib.parse import urlencode +from urllib.parse import parse_qs, urlsplit, urlunsplit import flask from userdatamodel.driver import SQLAlchemyDriver @@ -23,7 +23,7 @@ def random_str(length): - return "".join(rng.choice(alphanumeric) for _ in xrange(length)) + return "".join(rng.choice(alphanumeric) for _ in range(length)) def json_res(data): @@ -52,7 +52,9 @@ def create_client( hashed_secret = None if confidential: client_secret = random_str(55) - hashed_secret = bcrypt.hashpw(client_secret, bcrypt.gensalt()) + hashed_secret = bcrypt.hashpw( + client_secret.encode("utf-8"), bcrypt.gensalt() + ).decode("utf-8") auth_method = "client_secret_basic" if confidential else "none" with driver.session as s: user = query_for_user(session=s, username=username) @@ -99,7 +101,7 @@ def wrapper(*args, **kwargs): form["client_secret"] = bcrypt.hashpw( form["client_secret"].encode("utf-8"), client.client_secret.encode("utf-8"), - ) + ).decode("utf-8") flask.request.form = ImmutableMultiDict(form) return f(*args, **kwargs) @@ -129,7 +131,7 @@ def convert_key(d, converter): return d new = {} - for k, v in d.iteritems(): + for k, v in d.items(): new_v = v if isinstance(v, dict): new_v = convert_key(v, converter) @@ -147,7 +149,7 @@ def convert_value(d, converter): return converter(d) new = {} - for k, v in d.iteritems(): + for k, v in d.items(): new_v = v if isinstance(v, dict): new_v = convert_value(v, converter) @@ -174,7 +176,7 @@ def clear_cookies(response): """ Set all cookies to empty and expired. """ - for cookie_name in flask.request.cookies.keys(): + for cookie_name in list(flask.request.cookies.keys()): response.set_cookie(cookie_name, "", expires=0) @@ -198,7 +200,7 @@ def append_query_params(original_url, **kwargs): scheme, netloc, path, query_string, fragment = urlsplit(original_url) query_params = parse_qs(query_string) if kwargs is not None: - for key, value in kwargs.iteritems(): + for key, value in kwargs.items(): query_params[key] = [value] new_query_string = urlencode(query_params, doseq=True) diff --git a/requirements.txt b/requirements.txt index d981bde51..f5fc92f63 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,22 @@ Authlib==0.11 addict==2.1.1 -authutils==3.1.1 +authutils>=4.0.0<5.0.0 boto>=2.36.0<3.0.0 botocore>=1.7<1.10.39 boto3>=1.5<1.6 cached_property==1.5.1 cdislogging>=1.0.0<2.0.0 +cdiserrors==0.1.2 +cdispyutils==1.0.1 cryptography>=2.1.2<3.0 -enum34>=1.1.6 +datamodelutils==0.4.5 Flask==0.12.4 Flask-CORS==3.0.3 Flask_OAuthlib==0.9.4 flask-restful==0.3.6 Flask_SQLAlchemy_Session==1.1 gen3config==0.1.7 -gen3cirrus==0.4.0 +gen3cirrus==1.0.0 gen3users httplib2==0.10.3 markdown==3.1.1 @@ -32,12 +34,8 @@ setuptools==36.6.0 six==1.11.0 SQLAlchemy==1.3.3 temps==0.3.0 -userdatamodel==1.5.1 +userdatamodel==2.0.1 Werkzeug==0.12.2 pyyaml==5.1 retry==0.9.2 --e git+https://github.com/uc-cdis/flask-postgres-session.git@68bf5a9723a351729855c429eca8a0f4bbb830c7#egg=flask_postgres_session-0.1.3 --e git+https://git@github.com/uc-cdis/cdiserrors.git@0.1.0#egg=cdiserrors --e git+https://github.com/uc-cdis/storage-client.git@0.2.2#egg=storageclient-0.2.2 --e git+https://github.com/uc-cdis/datamodelutils.git@0.4.0#egg=datamodelutils --e git+https://github.com/uc-cdis/cdis-python-utils.git@0.2.10#egg=cdispyutils +git+https://github.com/uc-cdis/storage-client.git@1.0.0#egg=storageclient diff --git a/setup.py b/setup.py index 9467c3758..ccc4731f0 100644 --- a/setup.py +++ b/setup.py @@ -11,16 +11,14 @@ "botocore>=1.7,<1.9.0", "boto3>=1.5,<1.6", "cached_property>=1.5.1,<2.0.0", - "gen3config>=0.1.6,<1.0.0", - "gen3cirrus>=0.3.4,<1.0.0", "cryptography>=2.1.2<3.0", - "enum34>=1.1.6", "flask-restful>=0.3.6,<1.0.0", "Flask>=0.10.1,<1.0.0", "Flask-CORS>=3.0.3,<4.0.0", "Flask_OAuthlib>=0.9.4,<1.0.0", "Flask_SQLAlchemy_Session>=1.1,<2.0", - "gen3cirrus>=0.4.0,<1.0", + "gen3cirrus>=1.0.0,<2.0", + "gen3config>=0.1.6,<1.0.0", "google_api_python_client>=1.6.4,<2.0.0", "httplib2>=0.10.3,<1.0.0", "markdown>=3.1.1,<4.0.0", @@ -37,7 +35,7 @@ "six>=1.11.0,<2.0.0", "SQLAlchemy>=1.3.3,<1.4.0", "temps>=0.3.0,<1.0.0", - "userdatamodel~=1.5", + "userdatamodel>=2.0.1,<3.0.0", "Werkzeug>=0.12.2,<1.0.0", "storageclient", "pyyaml~=5.1", diff --git a/tests/admin/test_admin_projects.py b/tests/admin/test_admin_projects.py index b403f3894..458dadba0 100644 --- a/tests/admin/test_admin_projects.py +++ b/tests/admin/test_admin_projects.py @@ -20,16 +20,16 @@ def test_get_all_projects(db_session, awg_users): } expected = { "test_project_1": { - "auth_id": u"phs_project_1", + "auth_id": "phs_project_1", "associated buckets": [], "description": None, - "name": u"test_project_1", + "name": "test_project_1", }, "test_project_2": { - "auth_id": u"phs_project_2", + "auth_id": "phs_project_2", "associated buckets": [], "description": None, - "name": u"test_project_2", + "name": "test_project_2", }, } assert info == expected diff --git a/tests/admin/test_admin_users.py b/tests/admin/test_admin_users.py index f3b18aba7..44fbf01f7 100644 --- a/tests/admin/test_admin_users.py +++ b/tests/admin/test_admin_users.py @@ -206,6 +206,6 @@ def test_get_user_groups(db_session, awg_users): "projects": ["test_project_1"], }, ] - expected_groups.sort() - groups["groups"].sort() + expected_groups.sort(key=lambda x: x["name"]) + groups["groups"].sort(key=lambda x: x["name"]) assert groups["groups"] == expected_groups diff --git a/tests/admin/test_admin_users_endpoints.py b/tests/admin/test_admin_users_endpoints.py index 7416ad714..fbb657be6 100644 --- a/tests/admin/test_admin_users_endpoints.py +++ b/tests/admin/test_admin_users_endpoints.py @@ -4,13 +4,7 @@ import jwt import pytest -# Python 2 and 3 compatible -try: - from unittest.mock import Mock - from unittest.mock import patch -except ImportError: - from mock import Mock - from mock import patch +from unittest.mock import Mock, patch from fence.config import config from fence.models import ( @@ -63,7 +57,9 @@ def encoded_admin_jwt(kid, rsa_private_key): claims["sub"] = "5678" claims["iss"] = config["BASE_URL"] claims["exp"] += 600 - return jwt.encode(claims, key=rsa_private_key, headers=headers, algorithm="RS256") + return jwt.encode( + claims, key=rsa_private_key, headers=headers, algorithm="RS256" + ).decode("utf-8") # Dictionary for all these random magic numbers that the delete user diff --git a/tests/conftest.py b/tests/conftest.py index e8e83466b..12aac0026 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -92,7 +92,7 @@ def encoded_jwt(kid, rsa_private_key): headers = {"kid": kid} return jwt.encode( utils.default_claims(), key=rsa_private_key, headers=headers, algorithm="RS256" - ) + ).decode("utf-8") @pytest.fixture(scope="session") @@ -113,7 +113,7 @@ def encoded_jwt_expired(kid, rsa_private_key): claims_expired["iat"] -= 10000 return jwt.encode( claims_expired, key=rsa_private_key, headers=headers, algorithm="RS256" - ) + ).decode("utf-8") @pytest.fixture(scope="session") @@ -132,7 +132,7 @@ def encoded_jwt_refresh_token(claims_refresh, kid, rsa_private_key): headers = {"kid": kid} return jwt.encode( claims_refresh, key=rsa_private_key, headers=headers, algorithm="RS256" - ) + ).decode("utf-8") class Mocker(object): @@ -277,7 +277,7 @@ def app(kid, rsa_private_key, rsa_public_key): ) config.update(BASE_URL=config["BASE_URL"]) - config.update(ENCRYPTION_KEY=Fernet.generate_key()) + config.update(ENCRYPTION_KEY=Fernet.generate_key().decode("utf-8")) return fence.app @@ -856,7 +856,9 @@ def oauth_client(app, db_session, oauth_user): url = "https://oauth-test-client.net" client_id = "test-client" client_secret = fence.utils.random_str(50) - hashed_secret = bcrypt.hashpw(client_secret, bcrypt.gensalt()) + hashed_secret = bcrypt.hashpw( + client_secret.encode("utf-8"), bcrypt.gensalt() + ).decode("utf-8") test_user = db_session.query(models.User).filter_by(id=oauth_user.user_id).first() db_session.add( models.Client( @@ -884,7 +886,9 @@ def oauth_client_B(app, request, db_session): url = "https://oauth-test-client-B.net" client_id = "test-client-B" client_secret = fence.utils.random_str(50) - hashed_secret = bcrypt.hashpw(client_secret, bcrypt.gensalt()) + hashed_secret = bcrypt.hashpw( + client_secret.encode("utf-8"), bcrypt.gensalt() + ).decode("utf-8") test_user = db_session.query(models.User).filter_by(username="test").first() if not test_user: @@ -1092,7 +1096,7 @@ def encoded_creds_jwt( key=rsa_private_key, headers=headers, algorithm="RS256", - ), + ).decode("utf-8"), user_id=user_client["user_id"], client_id=oauth_client["client_id"], proxy_group_id=google_proxy_group["id"], @@ -1124,7 +1128,7 @@ def encoded_jwt_no_proxy_group(kid, rsa_private_key, user_client, oauth_client): key=rsa_private_key, headers=headers, algorithm="RS256", - ), + ).decode("utf-8"), user_id=user_client["user_id"], client_id=oauth_client["client_id"], ) diff --git a/tests/credentials/google/test_credentials.py b/tests/credentials/google/test_credentials.py index 66e139def..10970a9d1 100644 --- a/tests/credentials/google/test_credentials.py +++ b/tests/credentials/google/test_credentials.py @@ -21,13 +21,7 @@ from fence.config import config -# Python 2 and 3 compatible -try: - from unittest.mock import MagicMock - from unittest.mock import patch -except ImportError: - from mock import MagicMock - from mock import patch +from unittest.mock import MagicMock, patch def _populate_test_identity(session, **kwargs): @@ -368,7 +362,7 @@ def test_google_create_access_token_post( args, kwargs = ( cloud_manager.return_value.__enter__.return_value.get_access_key ).call_args - combined = [arg for arg in args] + [value for key, value in kwargs.iteritems()] + combined = [arg for arg in args] + [value for key, value in kwargs.items()] assert service_account_id in combined or service_account_email in combined assert response.status_code == 200 @@ -443,7 +437,7 @@ def get_account_keys(*args, **kwargs): args, kwargs = ( cloud_manager.return_value.__enter__.return_value.delete_service_account_key ).call_args - all_args = [arg for arg in args] + [value for key, value in kwargs.iteritems()] + all_args = [arg for arg in args] + [value for key, value in kwargs.items()] assert service_account_id in all_args or service_account_email in all_args assert service_account_key in all_args diff --git a/tests/data/test_data.py b/tests/data/test_data.py index 6b73e53e8..b902c86d7 100644 --- a/tests/data/test_data.py +++ b/tests/data/test_data.py @@ -1,6 +1,6 @@ import json import mock -import urlparse +import urllib.parse import uuid import jwt @@ -13,13 +13,7 @@ from tests import utils -# Python 2 and 3 compatible -try: - from unittest.mock import MagicMock - from unittest.mock import patch -except ImportError: - from mock import MagicMock - from mock import patch +from unittest.mock import MagicMock, patch @pytest.mark.parametrize( @@ -52,14 +46,14 @@ def test_indexd_download_file( key=rsa_private_key, headers={"kid": kid}, algorithm="RS256", - ) + ).decode("utf-8") } response = client.get(path, headers=headers, query_string=query_string) assert response.status_code == 200 - assert "url" in response.json.keys() + assert "url" in list(response.json.keys()) # defaults to signing url, check that it's not just raw url - assert urlparse.urlparse(response.json["url"]).query != "" + assert urllib.parse.urlparse(response.json["url"]).query != "" @pytest.mark.parametrize( @@ -91,11 +85,11 @@ def test_indexd_upload_file( key=rsa_private_key, headers={"kid": kid}, algorithm="RS256", - ) + ).decode("utf-8") } response = client.get(path, headers=headers) assert response.status_code == 200 - assert "url" in response.json.keys() + assert "url" in list(response.json.keys()) @pytest.mark.parametrize( @@ -127,11 +121,11 @@ def test_indexd_download_file_no_protocol( key=rsa_private_key, headers={"kid": kid}, algorithm="RS256", - ) + ).decode("utf-8") } response = client.get(path, headers=headers) assert response.status_code == 200 - assert "url" in response.json.keys() + assert "url" in list(response.json.keys()) def test_indexd_download_file_no_jwt(client, auth_client): @@ -198,7 +192,7 @@ def test_unauthorized_indexd_download_file( key=rsa_private_key, headers={"kid": kid}, algorithm="RS256", - ) + ).decode("utf-8") } response = client.get(path, headers=headers) assert response.status_code == 401 @@ -237,7 +231,7 @@ def test_unauthorized_indexd_upload_file( key=rsa_private_key, headers={"kid": kid}, algorithm="RS256", - ) + ).decode("utf-8") } response = client.get(path, headers=headers) assert response.status_code == 401 @@ -276,7 +270,7 @@ def test_unavailable_indexd_upload_file( key=rsa_private_key, headers={"kid": kid}, algorithm="RS256", - ) + ).decode("utf-8") } response = client.get(path, headers=headers) assert response.status_code == 401 @@ -304,10 +298,10 @@ def test_public_object_download_file( path = "/data/download/1" response = client.get(path) assert response.status_code == 200 - assert "url" in response.json.keys() + assert "url" in list(response.json.keys()) # defaults to signing url, check that it's not just raw url - assert urlparse.urlparse(response.json["url"]).query != "" + assert urllib.parse.urlparse(response.json["url"]).query != "" @pytest.mark.parametrize( @@ -328,10 +322,10 @@ def test_public_object_download_file_no_force_sign( path = "/data/download/1?no_force_sign=True" response = client.get(path) assert response.status_code == 200 - assert "url" in response.json.keys() + assert "url" in list(response.json.keys()) # make sure we honor no_force_sign, check that response is unsigned raw url - assert urlparse.urlparse(response.json["url"]).query == "" + assert urllib.parse.urlparse(response.json["url"]).query == "" @pytest.mark.parametrize( @@ -357,7 +351,7 @@ def test_public_bucket_download_file( # we should NOT sign AWS S3 urls if the bucket itself is public if not public_bucket_indexd_client.startswith("s3"): # defaults to signing url, check that it's not just raw url - assert urlparse.urlparse(response.json["url"]).query != "" + assert urllib.parse.urlparse(response.json["url"]).query != "" @pytest.mark.parametrize( @@ -381,7 +375,7 @@ def test_public_bucket_download_file_no_force_sign( assert response.json.get("url") # make sure we honor no_force_sign, check that response is unsigned raw url - assert urlparse.urlparse(response.json["url"]).query == "" + assert urllib.parse.urlparse(response.json["url"]).query == "" @pytest.mark.parametrize("public_bucket_indexd_client", ["s2"], indirect=True) @@ -594,11 +588,11 @@ def test_rbac( key=rsa_private_key, headers={"kid": kid}, algorithm="RS256", - ) + ).decode("utf-8") } response = client.get(path, headers=headers, query_string=query_string) assert response.status_code == 200 - assert "url" in response.json.keys() + assert "url" in list(response.json.keys()) mock_arborist_requests( {"arborist/auth/request": {"POST": ('{"auth": "false"}', 403)}} diff --git a/tests/dbgap_sync/conftest.py b/tests/dbgap_sync/conftest.py index b317d6d1d..25ce7f4fb 100644 --- a/tests/dbgap_sync/conftest.py +++ b/tests/dbgap_sync/conftest.py @@ -1,12 +1,6 @@ import os -# Python 2 and 3 compatible -try: - from unittest.mock import MagicMock - from unittest.mock import patch -except ImportError: - from mock import MagicMock - from mock import patch +from unittest.mock import MagicMock, patch from yaml import safe_load as yaml_load from cirrus import GoogleCloudManager diff --git a/tests/dbgap_sync/test_user_sync.py b/tests/dbgap_sync/test_user_sync.py index a7cfb024f..85c4692fe 100644 --- a/tests/dbgap_sync/test_user_sync.py +++ b/tests/dbgap_sync/test_user_sync.py @@ -319,7 +319,7 @@ def test_update_arborist(syncer, db_session): resource_to_parent_paths.setdefault(resource, []).append(parent_path) for resource in expect_resources: - assert resource in resource_to_parent_paths.keys() + assert resource in list(resource_to_parent_paths.keys()) if resource == "data_file": assert resource_to_parent_paths[resource] == ["/"] elif resource == project_with_mult_namespaces: diff --git a/tests/google/conftest.py b/tests/google/conftest.py index d743f3af7..bd3dbd71c 100644 --- a/tests/google/conftest.py +++ b/tests/google/conftest.py @@ -29,13 +29,7 @@ from fence.resources.google.validity import GoogleServiceAccountValidity -# Python 2 and 3 compatible -try: - from unittest.mock import MagicMock - from unittest.mock import patch -except ImportError: - from mock import MagicMock - from mock import patch +from unittest.mock import MagicMock, patch @pytest.fixture(scope="function") @@ -64,7 +58,7 @@ def encoded_jwt_service_accounts_access( key=rsa_private_key, headers=headers, algorithm="RS256", - ), + ).decode("utf-8"), user_id=user_client["user_id"], client_id=oauth_client["client_id"], ) diff --git a/tests/google/test_access_utils.py b/tests/google/test_access_utils.py index 1a546272d..72f9cf832 100644 --- a/tests/google/test_access_utils.py +++ b/tests/google/test_access_utils.py @@ -1,13 +1,7 @@ import pytest import time -# Python 2 and 3 compatible -try: - from unittest.mock import MagicMock - from unittest.mock import patch -except ImportError: - from mock import MagicMock - from mock import patch +from unittest.mock import MagicMock, patch from sqlalchemy import or_ from cirrus.errors import CirrusError @@ -332,8 +326,8 @@ def test_update_user_service_account_success(cloud_manager, db_session, setup_da project = db_session.query(Project).filter_by(auth_id="test_auth_1").first() project_ids = [ - project.id - for project in ( + item.project_id + for item in ( db_session.query(ServiceAccountAccessPrivilege) .filter_by(service_account_id=service_account.id) .all() diff --git a/tests/google/test_google_monitor.py b/tests/google/test_google_monitor.py index 363a25df4..ee75919d6 100644 --- a/tests/google/test_google_monitor.py +++ b/tests/google/test_google_monitor.py @@ -1,10 +1,4 @@ -# Python 2 and 3 compatible -try: - from unittest.mock import MagicMock - from unittest.mock import patch -except ImportError: - from mock import MagicMock - from mock import patch +from unittest.mock import MagicMock, patch import fence from fence.scripting.google_monitor import _get_users_without_access, validation_check diff --git a/tests/google/test_service_accounts.py b/tests/google/test_service_accounts.py index 1cb56baa4..9c5de8adc 100644 --- a/tests/google/test_service_accounts.py +++ b/tests/google/test_service_accounts.py @@ -58,7 +58,7 @@ import time from datetime import datetime from io import StringIO -from urllib import quote +from urllib.parse import quote from fence.models import ( Bucket, @@ -72,13 +72,7 @@ from fence.config import config -# Python 2 and 3 compatible -try: - from unittest.mock import MagicMock - from unittest.mock import patch -except ImportError: - from mock import MagicMock - from mock import patch, mock_open +from unittest.mock import MagicMock, patch, mock_open EXPECTED_ERROR_RESPONSE_KEYS = set(["status", "error", "error_description"]) @@ -119,7 +113,7 @@ def test_google_service_account_monitor( mock_path = patch("fence.blueprints.google.os", path_mock) # mock_path = patch('os.path.exists', True) - mocked_open = patch("__builtin__.open", mock_open(read_data=creds_file)) + mocked_open = patch("builtins.open", mock_open(read_data=creds_file)) monkeypatch.setitem(config, "CIRRUS_CFG", {"GOOGLE_APPLICATION_CREDENTIALS": "."}) @@ -1055,4 +1049,4 @@ def _assert_expected_error_response_structure(response, project_access): def _assert_expected_error_info_structure(data): - assert EXPECTED_ERROR_RESPONSE_KEYS.issubset(data.keys()) + assert EXPECTED_ERROR_RESPONSE_KEYS.issubset(list(data.keys())) diff --git a/tests/google/test_validity_info.py b/tests/google/test_validity_info.py index 07732da72..eebb35939 100644 --- a/tests/google/test_validity_info.py +++ b/tests/google/test_validity_info.py @@ -9,13 +9,7 @@ from fence.models import Project from fence.errors import NotFound -# Python 2 and 3 compatible -try: - from unittest.mock import MagicMock - from unittest.mock import patch -except ImportError: - from mock import MagicMock - from mock import patch +from unittest.mock import MagicMock, patch def test_dict_like_validity_object(): diff --git a/tests/jwt/conftest.py b/tests/jwt/conftest.py index ed8526176..fcfd33387 100644 --- a/tests/jwt/conftest.py +++ b/tests/jwt/conftest.py @@ -18,7 +18,7 @@ def encoded_jwt(kid, rsa_private_key): headers = {"kid": kid} return jwt.encode( utils.default_claims(), key=rsa_private_key, headers=headers, algorithm="RS256" - ) + ).decode("utf-8") @pytest.fixture(scope="session") @@ -41,7 +41,7 @@ def encoded_jwt_expired(claims, kid, rsa_private_key): claims_expired["iat"] -= 10000 return jwt.encode( claims_expired, key=rsa_private_key, headers=headers, algorithm="RS256" - ) + ).decode("utf-8") @pytest.fixture(scope="session") @@ -61,4 +61,4 @@ def encoded_jwt_refresh_token(claims_refresh, kid, rsa_private_key): headers = {"kid": kid} return jwt.encode( claims_refresh, key=rsa_private_key, headers=headers, algorithm="RS256" - ) + ).decode("utf-8") diff --git a/tests/link/conftest.py b/tests/link/conftest.py index 7c819b635..432dad6e4 100644 --- a/tests/link/conftest.py +++ b/tests/link/conftest.py @@ -1,12 +1,6 @@ import pytest -# Python 2 and 3 compatible -try: - from unittest.mock import MagicMock - from unittest.mock import patch -except ImportError: - from mock import MagicMock - from mock import patch +from unittest.mock import MagicMock, patch @pytest.fixture(scope="function") diff --git a/tests/link/test_link.py b/tests/link/test_link.py index 9841a1edd..8943cccbc 100644 --- a/tests/link/test_link.py +++ b/tests/link/test_link.py @@ -1,14 +1,8 @@ import flask import time -from urlparse import urlparse, parse_qs, urlunparse - -# Python 2 and 3 compatible -try: - from unittest.mock import MagicMock - from unittest.mock import patch -except ImportError: - from mock import MagicMock - from mock import patch +from urllib.parse import urlparse, parse_qs, urlunparse + +from unittest.mock import MagicMock, patch from fence.resources.storage.cdis_jwt import create_session_token from fence.config import config diff --git a/tests/oidc/core/authorization/test_id_token_hint.py b/tests/oidc/core/authorization/test_id_token_hint.py index 875519dd4..7ed50f68b 100644 --- a/tests/oidc/core/authorization/test_id_token_hint.py +++ b/tests/oidc/core/authorization/test_id_token_hint.py @@ -21,12 +21,7 @@ """ import pytest -try: - # Python 3 - from urllib.parse import urlparse, parse_qs -except ImportError: - # Python 2 - from urlparse import urlparse, parse_qs +from urllib.parse import urlparse, parse_qs from fence.jwt.validate import validate_jwt from tests.utils import oauth2 diff --git a/tests/oidc/core/authorization/test_prompt.py b/tests/oidc/core/authorization/test_prompt.py index 4a76e9798..1546901c2 100644 --- a/tests/oidc/core/authorization/test_prompt.py +++ b/tests/oidc/core/authorization/test_prompt.py @@ -35,18 +35,9 @@ from fence.config import config -# Python 2 and 3 compatible -try: - from unittest.mock import patch -except ImportError: - from mock import patch - -try: - # Python 3 - from urllib.parse import urlparse, parse_qs -except ImportError: - # Python 2 - from urlparse import urlparse, parse_qs +from unittest.mock import patch + +from urllib.parse import urlparse, parse_qs # Reasons for skipping tests. diff --git a/tests/oidc/core/token/test_token_response.py b/tests/oidc/core/token/test_token_response.py index 5a66b3ea6..fa635bfb2 100644 --- a/tests/oidc/core/token/test_token_response.py +++ b/tests/oidc/core/token/test_token_response.py @@ -51,8 +51,8 @@ def test_id_token_required_fields(token_response): assert type(id_token["exp"]) is int assert type(id_token["iat"]) is int - assert type(id_token["sub"]) is unicode - assert type(id_token["iss"]) is unicode + assert type(id_token["sub"]) is str + assert type(id_token["iss"]) is str assert type(id_token["aud"]) is list diff --git a/tests/privacy-policy/test_privacy_policy.py b/tests/privacy-policy/test_privacy_policy.py index 2944adef5..6e0bb5378 100644 --- a/tests/privacy-policy/test_privacy_policy.py +++ b/tests/privacy-policy/test_privacy_policy.py @@ -1,14 +1,14 @@ def test_markdown(client, privacy_policy_md): response = client.get("/privacy-policy", headers={"Accept": "text/markdown"}) assert response.status_code == 200 - assert response.data.startswith(privacy_policy_md) + assert response.data.startswith(bytes(privacy_policy_md, "utf-8")) def test_html(client, privacy_policy_html): response = client.get("/privacy-policy", headers={"Accept": "text/html"}) assert response.status_code == 200 - assert response.data.startswith(privacy_policy_html) + assert response.data.startswith(bytes(privacy_policy_html, "utf-8")) # also should default to HTML response = client.get("/privacy-policy", headers={"Accept": "*/*"}) assert response.status_code == 200 - assert response.data.startswith(privacy_policy_html) + assert response.data.startswith(bytes(privacy_policy_html, "utf-8")) diff --git a/tests/rbac/conftest.py b/tests/rbac/conftest.py index d014cf3d1..2a09758f3 100644 --- a/tests/rbac/conftest.py +++ b/tests/rbac/conftest.py @@ -3,11 +3,7 @@ the database to check the return from the RBAC endpoints. """ -# Python 2 and 3 compatible -try: - from unittest.mock import MagicMock -except ImportError: - from mock import MagicMock +from unittest.mock import MagicMock import pytest diff --git a/tests/rbac/test_client.py b/tests/rbac/test_client.py index 2e9931755..a997dade5 100644 --- a/tests/rbac/test_client.py +++ b/tests/rbac/test_client.py @@ -3,11 +3,7 @@ the correct URLs on the arborist API. """ -# Python 2 and 3 compatible -try: - from unittest import mock -except ImportError: - import mock +from unittest import mock from fence.rbac.client import ArboristClient diff --git a/tests/rfc7517/test_jwks.py b/tests/rfc7517/test_jwks.py index 7ed6bb6cd..7a74025f9 100644 --- a/tests/rfc7517/test_jwks.py +++ b/tests/rfc7517/test_jwks.py @@ -44,5 +44,5 @@ def test_response_values(app, client): assert key["key_ops"] == "verify" assert key["kid"] in app_kids # Attempt to reproduce the public key from the JWK response. - key_pem = jwk.construct(key).to_pem() + key_pem = jwk.construct(key).to_pem().decode("utf-8") assert key_pem in app_public_keys diff --git a/tests/scripting/test_fence-create.py b/tests/scripting/test_fence-create.py index 68d384824..dbc707c84 100644 --- a/tests/scripting/test_fence-create.py +++ b/tests/scripting/test_fence-create.py @@ -1,12 +1,7 @@ import time import mock -# Python 2 and 3 compatible -try: - from unittest.mock import patch -except ImportError: - from mock import patch -from mock import MagicMock +from unittest.mock import MagicMock, patch import pytest import cirrus @@ -134,7 +129,7 @@ def test_delete_users(app, db_session, example_usernames): # Get the list of usernames for users that still exist. # (The `list(zip(...))` trick is to turn a list of 1-tuples into a # flattened list.) - remaining_usernames = list(zip(*db_session.query(User.username).all())[0]) + remaining_usernames = list(next(zip(*db_session.query(User.username).all()))) assert example_usernames[0] in remaining_usernames for username in example_usernames[1:]: assert username not in remaining_usernames @@ -330,7 +325,7 @@ def test_delete_expired_service_accounts_with_one_fail_first( fence.settings = MagicMock() cirrus.config.update = MagicMock() cloud_manager.return_value.__enter__.return_value.remove_member_from_group.side_effect = [ - HttpError(mock.Mock(status=403), "Permission denied"), + HttpError(mock.Mock(status=403), bytes("Permission denied", "utf-8")), {}, ] _setup_service_account_to_google_bucket_access_group(db_session) @@ -381,7 +376,7 @@ def test_delete_expired_service_accounts_with_one_fail_second( fence.settings = MagicMock() cloud_manager.return_value.__enter__.return_value.remove_member_from_group.side_effect = [ {}, - HttpError(mock.Mock(status=403), "Permission denied"), + HttpError(mock.Mock(status=403), bytes("Permission denied", "utf-8")), ] _setup_service_account_to_google_bucket_access_group(db_session) service_accounts = db_session.query(UserServiceAccount).all() diff --git a/tests/session/test_session.py b/tests/session/test_session.py index eb4e0eeb9..fa40f2f84 100644 --- a/tests/session/test_session.py +++ b/tests/session/test_session.py @@ -8,15 +8,7 @@ from fence.jwt.keys import default_public_key from fence.jwt.validate import validate_jwt -# Python 2 and 3 compatible -try: - from unittest.mock import MagicMock - from unittest.mock import patch - from unittest.mock import call -except ImportError: - from mock import MagicMock - from mock import patch - from mock import call +from unittest.mock import MagicMock, patch, call import pytest @@ -275,9 +267,7 @@ def test_valid_session_valid_access_token_diff_user( def _get_cookies_from_response(response): raw_cookies = [ - header[1] - for header in response.headers.iteritems() - if header[0] == "Set-Cookie" + header[1] for header in response.headers.items() if header[0] == "Set-Cookie" ] cookies = {} for cookie in raw_cookies: diff --git a/tests/test_import_fence.py b/tests/test_import_fence.py index 3d711480f..acea5bef1 100644 --- a/tests/test_import_fence.py +++ b/tests/test_import_fence.py @@ -22,15 +22,14 @@ def reload_modules(module_name): # actually exist, even if it does. To do this, patch-delete the attribute # and reload all the fence modules. fence_submodules = [ - module for module in sys.modules.keys() if module.startswith(module_name) + module for module in list(sys.modules.keys()) if module.startswith(module_name) ] for module in fence_submodules: if sys.modules[module]: # SQLAlchemy gets upset when a table is loaded twice, so ignore # that. try: - # NOTE: in python3 this should become ``importlib.reload`` - reload(sys.modules[module]) + importlib.reload(sys.modules[module]) except InvalidRequestError: pass diff --git a/tests/test_logout.py b/tests/test_logout.py index 25825cdf7..7202b1f38 100644 --- a/tests/test_logout.py +++ b/tests/test_logout.py @@ -1,6 +1,5 @@ import mock -import urlparse -import urllib +import urllib.request, urllib.parse, urllib.error from fence.auth import build_redirect_url from fence.config import config @@ -42,9 +41,9 @@ def test_logout_itrust(client, db_session): r = client.get("/user/") r = client.get("/logout?next={}".format(redirect)) assert r.status_code == 302 - parsed_url = urlparse.urlparse(r.location) - raw_redirect = urlparse.parse_qs(parsed_url.query).get("AppReturnUrl")[0] - result_redirect = urllib.unquote(raw_redirect) + parsed_url = urllib.parse.urlparse(r.location) + raw_redirect = urllib.parse.parse_qs(parsed_url.query).get("AppReturnUrl")[0] + result_redirect = urllib.parse.unquote(raw_redirect) assert result_redirect == redirect @@ -77,7 +76,7 @@ def test_logout_fence(app, client, user_with_fence_provider, monkeypatch): assert r.status_code == 302 assert r.location.startswith(other_fence_logout_url) - parsed_url = urlparse.urlparse(r.location) - raw_redirect = urlparse.parse_qs(parsed_url.query).get("next")[0] - result_redirect = urllib.unquote(raw_redirect) + parsed_url = urllib.parse.urlparse(r.location) + raw_redirect = urllib.parse.parse_qs(parsed_url.query).get("next")[0] + result_redirect = urllib.parse.unquote(raw_redirect) assert result_redirect == redirect diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index 1bb2db4b4..a2a4e0a64 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -1,6 +1,6 @@ import os import time -import urlparse +import urllib.parse import uuid from flask import current_app @@ -34,7 +34,7 @@ def read_file(filename): def create_user(users, db_session, is_admin=False): s = db_session - for username in users.keys(): + for username in list(users.keys()): user = query_for_user(session=s, username=username) if not user: user = User(username=username, is_admin=is_admin) @@ -66,7 +66,7 @@ def create_user(users, db_session, is_admin=False): def create_awg_user(users, db_session): s = db_session - for username in users.keys(): + for username in list(users.keys()): user = query_for_user(session=s, username=username) if not user: user = User(username=username) @@ -137,7 +137,7 @@ def create_providers(data, db_session): s.add(prov) s.flush - for name, user in data["users"].items(): + for name, user in list(data["users"].items()): new_user = User() new_user.username = name new_user.email = user["email"] @@ -447,4 +447,4 @@ def remove_qs(url): """ Remove the query string from a url. """ - return urlparse.urljoin(url, urlparse.urlparse(url).path) + return urllib.parse.urljoin(url, urllib.parse.urlparse(url).path) diff --git a/tests/utils/oauth2/client.py b/tests/utils/oauth2/client.py index f8bf3ee2d..8e88ac17e 100644 --- a/tests/utils/oauth2/client.py +++ b/tests/utils/oauth2/client.py @@ -1,5 +1,5 @@ -import urllib -from urlparse import parse_qs, urlparse +import urllib.request, urllib.parse, urllib.error +from urllib.parse import parse_qs, urlparse import fence.utils @@ -56,7 +56,7 @@ def __init__(self, response): self.access_token = response.json.get("access_token") self.refresh_token = response.json.get("refresh_token") self.id_token = response.json.get("id_token") - except ValueError: + except (ValueError, AttributeError): self.access_token = None self.refresh_token = None self.id_token = None @@ -155,7 +155,7 @@ def _path_for_authorize(self, params=None): """ path = self.PATH_AUTHORIZE if params: - path += "?" + urllib.urlencode(query=params) + path += "?" + urllib.parse.urlencode(query=params) return path def authorize(self, method="POST", data=None, do_asserts=True, include_auth=True): diff --git a/wsgi.py b/wsgi.py deleted file mode 100644 index 6eb54cc89..000000000 --- a/wsgi.py +++ /dev/null @@ -1,5 +0,0 @@ -from fence import app_init -from fence import app - -app_init(app) -application = app