diff --git a/appengine/standard/conftest.py b/appengine/standard/conftest.py index 4fd85dcc9067..8a770af7a43d 100644 --- a/appengine/standard/conftest.py +++ b/appengine/standard/conftest.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + # Import py.test hooks and fixtures for App Engine from gcp.testing.appengine import ( login, @@ -19,9 +21,20 @@ pytest_runtest_call, run_tasks, testbed) +import six (login) (pytest_configure) (pytest_runtest_call) (run_tasks) (testbed) + + +def pytest_ignore_collect(path, config): + """Skip App Engine tests in python 3 or if no SDK is available.""" + if 'appengine/standard' in str(path): + if six.PY3: + return True + if 'GAE_SDK_PATH' not in os.environ: + return True + return False diff --git a/bigtable/hello/main.py b/bigtable/hello/main.py index 54c7f321239b..949c6baffbf5 100644 --- a/bigtable/hello/main.py +++ b/bigtable/hello/main.py @@ -70,7 +70,7 @@ def main(project_id, instance_id, table_id): row = table.row(row_key) row.set_cell( column_family_id, - column_id.encode('utf-8'), + column_id, value.encode('utf-8')) row.commit() # [END writing_rows] @@ -79,7 +79,7 @@ def main(project_id, instance_id, table_id): print('Getting a single greeting by row key.') key = 'greeting0' row = table.read_row(key.encode('utf-8')) - value = row.cells[column_family_id][column_id.encode('utf-8')][0].value + value = row.cells[column_family_id][column_id][0].value print('\t{}: {}'.format(key, value.decode('utf-8'))) # [END getting_a_row] @@ -90,7 +90,7 @@ def main(project_id, instance_id, table_id): for row_key, row in partial_rows.rows.items(): key = row_key.decode('utf-8') - cell = row.cells[column_family_id][column_id.encode('utf-8')][0] + cell = row.cells[column_family_id][column_id][0] value = cell.value.decode('utf-8') print('\t{}: {}'.format(key, value)) # [END scanning_all_rows] diff --git a/bigtable/hello/main_test.py b/bigtable/hello/main_test.py index 49f77e9f936a..0cc9fee9291d 100644 --- a/bigtable/hello/main_test.py +++ b/bigtable/hello/main_test.py @@ -13,21 +13,13 @@ # limitations under the License. import random -import re -import sys from main import main -import pytest - -TABLE_NAME_FORMAT = 'Hello-Bigtable-{}' +TABLE_NAME_FORMAT = 'hell-bigtable-system-tests-{}' TABLE_NAME_RANGE = 10000 -@pytest.mark.skipif( - sys.version_info >= (3, 0), - reason=("grpc doesn't yet support python3 " - 'https://github.com/grpc/grpc/issues/282')) def test_main(cloud_config, capsys): table_name = TABLE_NAME_FORMAT.format( random.randrange(TABLE_NAME_RANGE)) @@ -37,12 +29,10 @@ def test_main(cloud_config, capsys): table_name) out, _ = capsys.readouterr() - assert re.search( - re.compile(r'Creating the Hello-Bigtable-[0-9]+ table\.'), out) - assert re.search(re.compile(r'Writing some greetings to the table\.'), out) - assert re.search(re.compile(r'Getting a single greeting by row key.'), out) - assert re.search(re.compile(r'greeting0: Hello World!'), out) - assert re.search(re.compile(r'Scanning for all greetings'), out) - assert re.search(re.compile(r'greeting1: Hello Cloud Bigtable!'), out) - assert re.search( - re.compile(r'Deleting the Hello-Bigtable-[0-9]+ table\.'), out) + assert 'Creating the {} table.'.format(table_name) in out + assert 'Writing some greetings to the table.' in out + assert 'Getting a single greeting by row key.' in out + assert 'Hello World!' in out + assert 'Scanning for all greetings' in out + assert 'Hello Cloud Bigtable!' in out + assert 'Deleting the {} table.'.format(table_name) in out diff --git a/bigtable/hello_happybase/main.py b/bigtable/hello_happybase/main.py index b0e6ef63530c..519eedc0676e 100644 --- a/bigtable/hello_happybase/main.py +++ b/bigtable/hello_happybase/main.py @@ -77,16 +77,16 @@ def main(project_id, instance_id, table_name): # [START getting_a_row] print('Getting a single greeting by row key.') - key = 'greeting0' + key = 'greeting0'.encode('utf-8') row = table.row(key) - print('\t{}: {}'.format(key, row[column_name])) + print('\t{}: {}'.format(key, row[column_name.encode('utf-8')])) # [END getting_a_row] # [START scanning_all_rows] print('Scanning for all greetings:') for key, row in table.scan(): - print('\t{}: {}'.format(key, row[column_name])) + print('\t{}: {}'.format(key, row[column_name.encode('utf-8')])) # [END scanning_all_rows] # [START deleting_a_table] diff --git a/bigtable/hello_happybase/main_test.py b/bigtable/hello_happybase/main_test.py index 49f77e9f936a..6e58dac4c3e5 100644 --- a/bigtable/hello_happybase/main_test.py +++ b/bigtable/hello_happybase/main_test.py @@ -13,21 +13,13 @@ # limitations under the License. import random -import re -import sys from main import main -import pytest - -TABLE_NAME_FORMAT = 'Hello-Bigtable-{}' +TABLE_NAME_FORMAT = 'hello_happybase-system-tests-{}' TABLE_NAME_RANGE = 10000 -@pytest.mark.skipif( - sys.version_info >= (3, 0), - reason=("grpc doesn't yet support python3 " - 'https://github.com/grpc/grpc/issues/282')) def test_main(cloud_config, capsys): table_name = TABLE_NAME_FORMAT.format( random.randrange(TABLE_NAME_RANGE)) @@ -37,12 +29,10 @@ def test_main(cloud_config, capsys): table_name) out, _ = capsys.readouterr() - assert re.search( - re.compile(r'Creating the Hello-Bigtable-[0-9]+ table\.'), out) - assert re.search(re.compile(r'Writing some greetings to the table\.'), out) - assert re.search(re.compile(r'Getting a single greeting by row key.'), out) - assert re.search(re.compile(r'greeting0: Hello World!'), out) - assert re.search(re.compile(r'Scanning for all greetings'), out) - assert re.search(re.compile(r'greeting1: Hello Cloud Bigtable!'), out) - assert re.search( - re.compile(r'Deleting the Hello-Bigtable-[0-9]+ table\.'), out) + assert 'Creating the {} table.'.format(table_name) in out + assert 'Writing some greetings to the table.' in out + assert 'Getting a single greeting by row key.' in out + assert 'Hello World!' in out + assert 'Scanning for all greetings' in out + assert 'Hello Cloud Bigtable!' in out + assert 'Deleting the {} table.'.format(table_name) in out diff --git a/nox.py b/nox.py index d6458265b7b1..2c9e53b4fda6 100644 --- a/nox.py +++ b/nox.py @@ -30,7 +30,6 @@ """ import fnmatch -import itertools import os import subprocess import tempfile @@ -46,16 +45,6 @@ '-x', '--no-success-flaky-report', '--cov', '--cov-config', '.coveragerc', '--cov-append', '--cov-report='] -# Blacklists of samples to ingnore. -# Bigtable and Speech are disabled because they use gRPC, which does not yet -# support Python 3. See: https://github.com/grpc/grpc/issues/282 -TESTS_BLACKLIST = set(( - './appengine/standard', - './bigtable', - './speech', - './testing')) -APPENGINE_BLACKLIST = set() - # Libraries that only work on Python 2.7 PY27_ONLY_LIBRARIES = ['mysql-python'] @@ -132,8 +121,11 @@ def filter_samples(sample_dirs, changed_files): def setup_appengine(session): """Installs the App Engine SDK.""" # Install the app engine sdk and setup import paths. + if session.interpreter.startswith('python3'): + return + gae_root = os.environ.get('GAE_ROOT', tempfile.gettempdir()) - session.env['PYTHONPATH'] = os.path.join(gae_root, 'google_appengine') + session.env['GAE_SDK_PATH'] = os.path.join(gae_root, 'google_appengine') session.run('gcprepotools', 'download-appengine-sdk', gae_root) # Create a lib directory to prevent the GAE vendor library from @@ -143,8 +135,8 @@ def setup_appengine(session): def run_tests_in_sesssion( - session, interpreter, use_appengine=False, skip_flaky=False, - changed_only=False, sample_directories=None): + session, interpreter, sample_directories, use_appengine=True, + skip_flaky=False, changed_only=False): """This is the main function for executing tests. It: @@ -173,13 +165,6 @@ def run_tests_in_sesssion( if skip_flaky: pytest_args.append('-m not slow and not flaky') - # session.posargs is any leftover arguments from the command line, - # which allows users to run a particular test instead of all of them. - if session.posargs: - sample_directories = session.posargs - elif sample_directories is None: - sample_directories = collect_sample_dirs('.', TESTS_BLACKLIST) - if changed_only: changed_files = get_changed_files() sample_directories = filter_samples( @@ -204,43 +189,38 @@ def run_tests_in_sesssion( @nox.parametrize('interpreter', ['python2.7', 'python3.4']) def session_tests(session, interpreter): - """Runs tests""" - run_tests_in_sesssion(session, interpreter) - + """Runs tests for all non-gae standard samples.""" + # session.posargs is any leftover arguments from the command line, + # which allows users to run a particular test instead of all of them. + sample_directories = session.posargs + if not sample_directories: + sample_directories = collect_sample_dirs('.') -def session_gae(session): - """Runs test for GAE Standard samples.""" run_tests_in_sesssion( - session, 'python2.7', use_appengine=True, - sample_directories=collect_sample_dirs( - 'appengine/standard', - APPENGINE_BLACKLIST)) + session, interpreter, sample_directories) -def session_grpc(session): - """Runs tests for samples that need grpc.""" - # TODO: Remove this when grpc supports Python 3. +def session_gae(session): + """Runs test for GAE Standard samples.""" + sample_directories = collect_sample_dirs('appengine/standard') run_tests_in_sesssion( - session, - 'python2.7', - sample_directories=itertools.chain( - collect_sample_dirs('speech'), - collect_sample_dirs('bigtable'))) + session, 'python2.7', sample_directories, use_appengine=True) @nox.parametrize('subsession', ['gae', 'tests']) def session_travis(session, subsession): """On travis, just run with python3.4 and don't run slow or flaky tests.""" if subsession == 'tests': + sample_directories = collect_sample_dirs( + '.', set('./appengine/standard')) run_tests_in_sesssion( - session, 'python3.4', skip_flaky=True, changed_only=True) + session, 'python3.4', sample_directories, + skip_flaky=True, changed_only=True) else: + sample_directories = collect_sample_dirs('appengine/standard') run_tests_in_sesssion( - session, 'python2.7', use_appengine=True, skip_flaky=True, - changed_only=True, - sample_directories=collect_sample_dirs( - 'appengine/standard', - APPENGINE_BLACKLIST)) + session, 'python2.7', sample_directories, + skip_flaky=True, changed_only=True) def session_lint(session): diff --git a/speech/api/speech_async_grpc_test.py b/speech/api/speech_async_grpc_test.py index 61070326bfac..4329bba8ff8b 100644 --- a/speech/api/speech_async_grpc_test.py +++ b/speech/api/speech_async_grpc_test.py @@ -13,17 +13,12 @@ import argparse import re -import sys import pytest from speech_async_grpc import _gcs_uri from speech_async_grpc import main -@pytest.mark.skipif( - sys.version_info >= (3, 0), - reason=("grpc doesn't yet support python3 " - 'https://github.com/grpc/grpc/issues/282')) def test_main(cloud_config, capsys): input_uri = 'gs://{}/speech/audio.flac'.format(cloud_config.storage_bucket) diff --git a/speech/api/speech_grpc_test.py b/speech/api/speech_grpc_test.py index 8b0ae9b6f61d..9e9eb7915c7c 100644 --- a/speech/api/speech_grpc_test.py +++ b/speech/api/speech_grpc_test.py @@ -12,17 +12,12 @@ # limitations under the License. import re -import sys import pytest from speech_grpc import _gcs_uri from speech_grpc import main -@pytest.mark.skipif( - sys.version_info >= (3, 0), - reason=("grpc doesn't yet support python3 " - 'https://github.com/grpc/grpc/issues/282')) def test_main(cloud_config, capsys): input_uri = 'gs://{}/speech/audio.flac'.format(cloud_config.storage_bucket) diff --git a/speech/api/speech_streaming_test.py b/speech/api/speech_streaming_test.py index e3b71b88ac99..b81264fca8fe 100644 --- a/speech/api/speech_streaming_test.py +++ b/speech/api/speech_streaming_test.py @@ -14,11 +14,8 @@ import contextlib import io import re -import sys import time -import pytest - import speech_streaming @@ -57,10 +54,6 @@ def mock_audio_stream(channels, rate, chunk): return mock_audio_stream -@pytest.mark.skipif( - sys.version_info >= (3, 0), - reason=("grpc doesn't yet support python3 " - 'https://github.com/grpc/grpc/issues/282')) def test_main(resource, monkeypatch, capsys): monkeypatch.setattr( speech_streaming, 'record_audio',