From d062c2ef698112944b9c096034b34e4618992401 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Wed, 17 Aug 2016 14:09:41 -0700 Subject: [PATCH 1/4] Remove grpc-python3 hackiness Change-Id: I6bf9a8acb9ba7d067b3095b5857094cbc322ff58 --- bigtable/hello/main.py | 6 +-- bigtable/hello/main_test.py | 26 ++++------- bigtable/hello_happybase/main.py | 6 +-- bigtable/hello_happybase/main_test.py | 26 ++++------- nox.py | 64 +++++++++------------------ speech/api/speech_async_grpc_test.py | 5 --- speech/api/speech_grpc_test.py | 5 --- speech/api/speech_streaming_test.py | 7 --- 8 files changed, 43 insertions(+), 102 deletions(-) diff --git a/bigtable/hello/main.py b/bigtable/hello/main.py index 54c7f321239b..949c6baffbf5 100644 --- a/bigtable/hello/main.py +++ b/bigtable/hello/main.py @@ -70,7 +70,7 @@ def main(project_id, instance_id, table_id): row = table.row(row_key) row.set_cell( column_family_id, - column_id.encode('utf-8'), + column_id, value.encode('utf-8')) row.commit() # [END writing_rows] @@ -79,7 +79,7 @@ def main(project_id, instance_id, table_id): print('Getting a single greeting by row key.') key = 'greeting0' row = table.read_row(key.encode('utf-8')) - value = row.cells[column_family_id][column_id.encode('utf-8')][0].value + value = row.cells[column_family_id][column_id][0].value print('\t{}: {}'.format(key, value.decode('utf-8'))) # [END getting_a_row] @@ -90,7 +90,7 @@ def main(project_id, instance_id, table_id): for row_key, row in partial_rows.rows.items(): key = row_key.decode('utf-8') - cell = row.cells[column_family_id][column_id.encode('utf-8')][0] + cell = row.cells[column_family_id][column_id][0] value = cell.value.decode('utf-8') print('\t{}: {}'.format(key, value)) # [END scanning_all_rows] diff --git a/bigtable/hello/main_test.py b/bigtable/hello/main_test.py index 49f77e9f936a..0cc9fee9291d 100644 --- a/bigtable/hello/main_test.py +++ b/bigtable/hello/main_test.py @@ -13,21 +13,13 @@ # limitations under the License. import random -import re -import sys from main import main -import pytest - -TABLE_NAME_FORMAT = 'Hello-Bigtable-{}' +TABLE_NAME_FORMAT = 'hell-bigtable-system-tests-{}' TABLE_NAME_RANGE = 10000 -@pytest.mark.skipif( - sys.version_info >= (3, 0), - reason=("grpc doesn't yet support python3 " - 'https://github.com/grpc/grpc/issues/282')) def test_main(cloud_config, capsys): table_name = TABLE_NAME_FORMAT.format( random.randrange(TABLE_NAME_RANGE)) @@ -37,12 +29,10 @@ def test_main(cloud_config, capsys): table_name) out, _ = capsys.readouterr() - assert re.search( - re.compile(r'Creating the Hello-Bigtable-[0-9]+ table\.'), out) - assert re.search(re.compile(r'Writing some greetings to the table\.'), out) - assert re.search(re.compile(r'Getting a single greeting by row key.'), out) - assert re.search(re.compile(r'greeting0: Hello World!'), out) - assert re.search(re.compile(r'Scanning for all greetings'), out) - assert re.search(re.compile(r'greeting1: Hello Cloud Bigtable!'), out) - assert re.search( - re.compile(r'Deleting the Hello-Bigtable-[0-9]+ table\.'), out) + assert 'Creating the {} table.'.format(table_name) in out + assert 'Writing some greetings to the table.' in out + assert 'Getting a single greeting by row key.' in out + assert 'Hello World!' in out + assert 'Scanning for all greetings' in out + assert 'Hello Cloud Bigtable!' in out + assert 'Deleting the {} table.'.format(table_name) in out diff --git a/bigtable/hello_happybase/main.py b/bigtable/hello_happybase/main.py index b0e6ef63530c..519eedc0676e 100644 --- a/bigtable/hello_happybase/main.py +++ b/bigtable/hello_happybase/main.py @@ -77,16 +77,16 @@ def main(project_id, instance_id, table_name): # [START getting_a_row] print('Getting a single greeting by row key.') - key = 'greeting0' + key = 'greeting0'.encode('utf-8') row = table.row(key) - print('\t{}: {}'.format(key, row[column_name])) + print('\t{}: {}'.format(key, row[column_name.encode('utf-8')])) # [END getting_a_row] # [START scanning_all_rows] print('Scanning for all greetings:') for key, row in table.scan(): - print('\t{}: {}'.format(key, row[column_name])) + print('\t{}: {}'.format(key, row[column_name.encode('utf-8')])) # [END scanning_all_rows] # [START deleting_a_table] diff --git a/bigtable/hello_happybase/main_test.py b/bigtable/hello_happybase/main_test.py index 49f77e9f936a..6e58dac4c3e5 100644 --- a/bigtable/hello_happybase/main_test.py +++ b/bigtable/hello_happybase/main_test.py @@ -13,21 +13,13 @@ # limitations under the License. import random -import re -import sys from main import main -import pytest - -TABLE_NAME_FORMAT = 'Hello-Bigtable-{}' +TABLE_NAME_FORMAT = 'hello_happybase-system-tests-{}' TABLE_NAME_RANGE = 10000 -@pytest.mark.skipif( - sys.version_info >= (3, 0), - reason=("grpc doesn't yet support python3 " - 'https://github.com/grpc/grpc/issues/282')) def test_main(cloud_config, capsys): table_name = TABLE_NAME_FORMAT.format( random.randrange(TABLE_NAME_RANGE)) @@ -37,12 +29,10 @@ def test_main(cloud_config, capsys): table_name) out, _ = capsys.readouterr() - assert re.search( - re.compile(r'Creating the Hello-Bigtable-[0-9]+ table\.'), out) - assert re.search(re.compile(r'Writing some greetings to the table\.'), out) - assert re.search(re.compile(r'Getting a single greeting by row key.'), out) - assert re.search(re.compile(r'greeting0: Hello World!'), out) - assert re.search(re.compile(r'Scanning for all greetings'), out) - assert re.search(re.compile(r'greeting1: Hello Cloud Bigtable!'), out) - assert re.search( - re.compile(r'Deleting the Hello-Bigtable-[0-9]+ table\.'), out) + assert 'Creating the {} table.'.format(table_name) in out + assert 'Writing some greetings to the table.' in out + assert 'Getting a single greeting by row key.' in out + assert 'Hello World!' in out + assert 'Scanning for all greetings' in out + assert 'Hello Cloud Bigtable!' in out + assert 'Deleting the {} table.'.format(table_name) in out diff --git a/nox.py b/nox.py index d6458265b7b1..f2076c171711 100644 --- a/nox.py +++ b/nox.py @@ -30,7 +30,6 @@ """ import fnmatch -import itertools import os import subprocess import tempfile @@ -46,16 +45,6 @@ '-x', '--no-success-flaky-report', '--cov', '--cov-config', '.coveragerc', '--cov-append', '--cov-report='] -# Blacklists of samples to ingnore. -# Bigtable and Speech are disabled because they use gRPC, which does not yet -# support Python 3. See: https://github.com/grpc/grpc/issues/282 -TESTS_BLACKLIST = set(( - './appengine/standard', - './bigtable', - './speech', - './testing')) -APPENGINE_BLACKLIST = set() - # Libraries that only work on Python 2.7 PY27_ONLY_LIBRARIES = ['mysql-python'] @@ -143,8 +132,8 @@ def setup_appengine(session): def run_tests_in_sesssion( - session, interpreter, use_appengine=False, skip_flaky=False, - changed_only=False, sample_directories=None): + session, interpreter, sample_directories, use_appengine=False, + skip_flaky=False, changed_only=False): """This is the main function for executing tests. It: @@ -173,13 +162,6 @@ def run_tests_in_sesssion( if skip_flaky: pytest_args.append('-m not slow and not flaky') - # session.posargs is any leftover arguments from the command line, - # which allows users to run a particular test instead of all of them. - if session.posargs: - sample_directories = session.posargs - elif sample_directories is None: - sample_directories = collect_sample_dirs('.', TESTS_BLACKLIST) - if changed_only: changed_files = get_changed_files() sample_directories = filter_samples( @@ -204,43 +186,39 @@ def run_tests_in_sesssion( @nox.parametrize('interpreter', ['python2.7', 'python3.4']) def session_tests(session, interpreter): - """Runs tests""" - run_tests_in_sesssion(session, interpreter) + """Runs tests for all non-gae standard samples.""" + # session.posargs is any leftover arguments from the command line, + # which allows users to run a particular test instead of all of them. + if session.posargs: + sample_directories = session.posargs + elif sample_directories is None: + sample_directories = collect_sample_dirs( + '.', set('./appengine/standard')) + + run_tests_in_sesssion(session, interpreter, sample_directories) def session_gae(session): """Runs test for GAE Standard samples.""" + sample_directories = collect_sample_dirs('appengine/standard') run_tests_in_sesssion( - session, 'python2.7', use_appengine=True, - sample_directories=collect_sample_dirs( - 'appengine/standard', - APPENGINE_BLACKLIST)) - - -def session_grpc(session): - """Runs tests for samples that need grpc.""" - # TODO: Remove this when grpc supports Python 3. - run_tests_in_sesssion( - session, - 'python2.7', - sample_directories=itertools.chain( - collect_sample_dirs('speech'), - collect_sample_dirs('bigtable'))) + session, 'python2.7', sample_directories, use_appengine=True) @nox.parametrize('subsession', ['gae', 'tests']) def session_travis(session, subsession): """On travis, just run with python3.4 and don't run slow or flaky tests.""" if subsession == 'tests': + sample_directories = collect_sample_dirs( + '.', set('./appengine/standard')) run_tests_in_sesssion( - session, 'python3.4', skip_flaky=True, changed_only=True) + session, 'python3.4', sample_directories, skip_flaky=True, + changed_only=True) else: + sample_directories = collect_sample_dirs('appengine/standard') run_tests_in_sesssion( - session, 'python2.7', use_appengine=True, skip_flaky=True, - changed_only=True, - sample_directories=collect_sample_dirs( - 'appengine/standard', - APPENGINE_BLACKLIST)) + session, 'python2.7', sample_directories, use_appengine=True, + skip_flaky=True, changed_only=True) def session_lint(session): diff --git a/speech/api/speech_async_grpc_test.py b/speech/api/speech_async_grpc_test.py index 61070326bfac..4329bba8ff8b 100644 --- a/speech/api/speech_async_grpc_test.py +++ b/speech/api/speech_async_grpc_test.py @@ -13,17 +13,12 @@ import argparse import re -import sys import pytest from speech_async_grpc import _gcs_uri from speech_async_grpc import main -@pytest.mark.skipif( - sys.version_info >= (3, 0), - reason=("grpc doesn't yet support python3 " - 'https://github.com/grpc/grpc/issues/282')) def test_main(cloud_config, capsys): input_uri = 'gs://{}/speech/audio.flac'.format(cloud_config.storage_bucket) diff --git a/speech/api/speech_grpc_test.py b/speech/api/speech_grpc_test.py index 8b0ae9b6f61d..9e9eb7915c7c 100644 --- a/speech/api/speech_grpc_test.py +++ b/speech/api/speech_grpc_test.py @@ -12,17 +12,12 @@ # limitations under the License. import re -import sys import pytest from speech_grpc import _gcs_uri from speech_grpc import main -@pytest.mark.skipif( - sys.version_info >= (3, 0), - reason=("grpc doesn't yet support python3 " - 'https://github.com/grpc/grpc/issues/282')) def test_main(cloud_config, capsys): input_uri = 'gs://{}/speech/audio.flac'.format(cloud_config.storage_bucket) diff --git a/speech/api/speech_streaming_test.py b/speech/api/speech_streaming_test.py index e3b71b88ac99..b81264fca8fe 100644 --- a/speech/api/speech_streaming_test.py +++ b/speech/api/speech_streaming_test.py @@ -14,11 +14,8 @@ import contextlib import io import re -import sys import time -import pytest - import speech_streaming @@ -57,10 +54,6 @@ def mock_audio_stream(channels, rate, chunk): return mock_audio_stream -@pytest.mark.skipif( - sys.version_info >= (3, 0), - reason=("grpc doesn't yet support python3 " - 'https://github.com/grpc/grpc/issues/282')) def test_main(resource, monkeypatch, capsys): monkeypatch.setattr( speech_streaming, 'record_audio', From a25007b8543157f03944c90a70880cb543d53fdd Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Wed, 17 Aug 2016 14:56:21 -0700 Subject: [PATCH 2/4] Unifiy app engine and python2.7 tests. Change-Id: Ie0df6747050035b2ef5f937951d5ff955073e6d4 --- appengine/standard/conftest.py | 15 +++++++++++++++ nox.py | 24 +++++++++++++----------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/appengine/standard/conftest.py b/appengine/standard/conftest.py index 4fd85dcc9067..f8e5c3261e7a 100644 --- a/appengine/standard/conftest.py +++ b/appengine/standard/conftest.py @@ -12,6 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + +import six + + # Import py.test hooks and fixtures for App Engine from gcp.testing.appengine import ( login, @@ -25,3 +30,13 @@ (pytest_runtest_call) (run_tasks) (testbed) + + +def pytest_ignore_collect(path, config): + """Skip App Engine tests in python 3 and if no SDK is available.""" + if 'appengine/standard' in str(path): + if six.PY3: + return True + if 'GAE_SDK_PATH' not in os.environ: + return True + return False diff --git a/nox.py b/nox.py index f2076c171711..31e38ac20abb 100644 --- a/nox.py +++ b/nox.py @@ -121,8 +121,11 @@ def filter_samples(sample_dirs, changed_files): def setup_appengine(session): """Installs the App Engine SDK.""" # Install the app engine sdk and setup import paths. + if session.interpreter.startswith('python3'): + return + gae_root = os.environ.get('GAE_ROOT', tempfile.gettempdir()) - session.env['PYTHONPATH'] = os.path.join(gae_root, 'google_appengine') + session.env['GAE_SDK_PATH'] = os.path.join(gae_root, 'google_appengine') session.run('gcprepotools', 'download-appengine-sdk', gae_root) # Create a lib directory to prevent the GAE vendor library from @@ -132,7 +135,7 @@ def setup_appengine(session): def run_tests_in_sesssion( - session, interpreter, sample_directories, use_appengine=False, + session, interpreter, sample_directories, use_appengine=True, skip_flaky=False, changed_only=False): """This is the main function for executing tests. @@ -189,13 +192,12 @@ def session_tests(session, interpreter): """Runs tests for all non-gae standard samples.""" # session.posargs is any leftover arguments from the command line, # which allows users to run a particular test instead of all of them. - if session.posargs: - sample_directories = session.posargs - elif sample_directories is None: - sample_directories = collect_sample_dirs( - '.', set('./appengine/standard')) + sample_directories = session.posargs + if not sample_directories: + sample_directories = collect_sample_dirs('.') - run_tests_in_sesssion(session, interpreter, sample_directories) + run_tests_in_sesssion( + session, interpreter, sample_directories, skip_flaky=True) def session_gae(session): @@ -212,12 +214,12 @@ def session_travis(session, subsession): sample_directories = collect_sample_dirs( '.', set('./appengine/standard')) run_tests_in_sesssion( - session, 'python3.4', sample_directories, skip_flaky=True, - changed_only=True) + session, 'python3.4', sample_directories, + skip_flaky=True, changed_only=True) else: sample_directories = collect_sample_dirs('appengine/standard') run_tests_in_sesssion( - session, 'python2.7', sample_directories, use_appengine=True, + session, 'python2.7', sample_directories, skip_flaky=True, changed_only=True) From 0952e8944280fe5bae1f4fcb02d37eeb47177017 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Wed, 17 Aug 2016 15:00:57 -0700 Subject: [PATCH 3/4] Don't skip flaky with session tests Change-Id: I0d3bdf3d6842339d04abc4ee6ddb26b8f44be3e5 --- nox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nox.py b/nox.py index 31e38ac20abb..2c9e53b4fda6 100644 --- a/nox.py +++ b/nox.py @@ -197,7 +197,7 @@ def session_tests(session, interpreter): sample_directories = collect_sample_dirs('.') run_tests_in_sesssion( - session, interpreter, sample_directories, skip_flaky=True) + session, interpreter, sample_directories) def session_gae(session): From b8792d9164f669133032eb26ab78281acb17c9c5 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 18 Aug 2016 10:10:46 -0700 Subject: [PATCH 4/4] Fix lint issue and review comments Change-Id: I02a53961b6411247ef06d84dad7b533cb97d89f7 --- appengine/standard/conftest.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/appengine/standard/conftest.py b/appengine/standard/conftest.py index f8e5c3261e7a..8a770af7a43d 100644 --- a/appengine/standard/conftest.py +++ b/appengine/standard/conftest.py @@ -14,9 +14,6 @@ import os -import six - - # Import py.test hooks and fixtures for App Engine from gcp.testing.appengine import ( login, @@ -24,6 +21,7 @@ pytest_runtest_call, run_tasks, testbed) +import six (login) (pytest_configure) @@ -33,7 +31,7 @@ def pytest_ignore_collect(path, config): - """Skip App Engine tests in python 3 and if no SDK is available.""" + """Skip App Engine tests in python 3 or if no SDK is available.""" if 'appengine/standard' in str(path): if six.PY3: return True