From d062c2ef698112944b9c096034b34e4618992401 Mon Sep 17 00:00:00 2001
From: Jon Wayne Parrott <jonwayne@google.com>
Date: Wed, 17 Aug 2016 14:09:41 -0700
Subject: [PATCH 1/4] Remove grpc-python3 hackiness
Change-Id: I6bf9a8acb9ba7d067b3095b5857094cbc322ff58
---
bigtable/hello/main.py | 6 +--
bigtable/hello/main_test.py | 26 ++++-------
bigtable/hello_happybase/main.py | 6 +--
bigtable/hello_happybase/main_test.py | 26 ++++-------
nox.py | 64 +++++++++------------------
speech/api/speech_async_grpc_test.py | 5 ---
speech/api/speech_grpc_test.py | 5 ---
speech/api/speech_streaming_test.py | 7 ---
8 files changed, 43 insertions(+), 102 deletions(-)
diff --git a/bigtable/hello/main.py b/bigtable/hello/main.py
index 54c7f321239b..949c6baffbf5 100644
--- a/bigtable/hello/main.py
+++ b/bigtable/hello/main.py
@@ -70,7 +70,7 @@ def main(project_id, instance_id, table_id):
row = table.row(row_key)
row.set_cell(
column_family_id,
- column_id.encode('utf-8'),
+ column_id,
value.encode('utf-8'))
row.commit()
# [END writing_rows]
@@ -79,7 +79,7 @@ def main(project_id, instance_id, table_id):
print('Getting a single greeting by row key.')
key = 'greeting0'
row = table.read_row(key.encode('utf-8'))
- value = row.cells[column_family_id][column_id.encode('utf-8')][0].value
+ value = row.cells[column_family_id][column_id][0].value
print('\t{}: {}'.format(key, value.decode('utf-8')))
# [END getting_a_row]
@@ -90,7 +90,7 @@ def main(project_id, instance_id, table_id):
for row_key, row in partial_rows.rows.items():
key = row_key.decode('utf-8')
- cell = row.cells[column_family_id][column_id.encode('utf-8')][0]
+ cell = row.cells[column_family_id][column_id][0]
value = cell.value.decode('utf-8')
print('\t{}: {}'.format(key, value))
# [END scanning_all_rows]
diff --git a/bigtable/hello/main_test.py b/bigtable/hello/main_test.py
index 49f77e9f936a..0cc9fee9291d 100644
--- a/bigtable/hello/main_test.py
+++ b/bigtable/hello/main_test.py
@@ -13,21 +13,13 @@
# limitations under the License.
import random
-import re
-import sys
from main import main
-import pytest
-
-TABLE_NAME_FORMAT = 'Hello-Bigtable-{}'
+TABLE_NAME_FORMAT = 'hell-bigtable-system-tests-{}'
TABLE_NAME_RANGE = 10000
-@pytest.mark.skipif(
- sys.version_info >= (3, 0),
- reason=("grpc doesn't yet support python3 "
- 'https://github.com/grpc/grpc/issues/282'))
def test_main(cloud_config, capsys):
table_name = TABLE_NAME_FORMAT.format(
random.randrange(TABLE_NAME_RANGE))
@@ -37,12 +29,10 @@ def test_main(cloud_config, capsys):
table_name)
out, _ = capsys.readouterr()
- assert re.search(
- re.compile(r'Creating the Hello-Bigtable-[0-9]+ table\.'), out)
- assert re.search(re.compile(r'Writing some greetings to the table\.'), out)
- assert re.search(re.compile(r'Getting a single greeting by row key.'), out)
- assert re.search(re.compile(r'greeting0: Hello World!'), out)
- assert re.search(re.compile(r'Scanning for all greetings'), out)
- assert re.search(re.compile(r'greeting1: Hello Cloud Bigtable!'), out)
- assert re.search(
- re.compile(r'Deleting the Hello-Bigtable-[0-9]+ table\.'), out)
+ assert 'Creating the {} table.'.format(table_name) in out
+ assert 'Writing some greetings to the table.' in out
+ assert 'Getting a single greeting by row key.' in out
+ assert 'Hello World!' in out
+ assert 'Scanning for all greetings' in out
+ assert 'Hello Cloud Bigtable!' in out
+ assert 'Deleting the {} table.'.format(table_name) in out
diff --git a/bigtable/hello_happybase/main.py b/bigtable/hello_happybase/main.py
index b0e6ef63530c..519eedc0676e 100644
--- a/bigtable/hello_happybase/main.py
+++ b/bigtable/hello_happybase/main.py
@@ -77,16 +77,16 @@ def main(project_id, instance_id, table_name):
# [START getting_a_row]
print('Getting a single greeting by row key.')
- key = 'greeting0'
+ key = 'greeting0'.encode('utf-8')
row = table.row(key)
- print('\t{}: {}'.format(key, row[column_name]))
+ print('\t{}: {}'.format(key, row[column_name.encode('utf-8')]))
# [END getting_a_row]
# [START scanning_all_rows]
print('Scanning for all greetings:')
for key, row in table.scan():
- print('\t{}: {}'.format(key, row[column_name]))
+ print('\t{}: {}'.format(key, row[column_name.encode('utf-8')]))
# [END scanning_all_rows]
# [START deleting_a_table]
diff --git a/bigtable/hello_happybase/main_test.py b/bigtable/hello_happybase/main_test.py
index 49f77e9f936a..6e58dac4c3e5 100644
--- a/bigtable/hello_happybase/main_test.py
+++ b/bigtable/hello_happybase/main_test.py
@@ -13,21 +13,13 @@
# limitations under the License.
import random
-import re
-import sys
from main import main
-import pytest
-
-TABLE_NAME_FORMAT = 'Hello-Bigtable-{}'
+TABLE_NAME_FORMAT = 'hello_happybase-system-tests-{}'
TABLE_NAME_RANGE = 10000
-@pytest.mark.skipif(
- sys.version_info >= (3, 0),
- reason=("grpc doesn't yet support python3 "
- 'https://github.com/grpc/grpc/issues/282'))
def test_main(cloud_config, capsys):
table_name = TABLE_NAME_FORMAT.format(
random.randrange(TABLE_NAME_RANGE))
@@ -37,12 +29,10 @@ def test_main(cloud_config, capsys):
table_name)
out, _ = capsys.readouterr()
- assert re.search(
- re.compile(r'Creating the Hello-Bigtable-[0-9]+ table\.'), out)
- assert re.search(re.compile(r'Writing some greetings to the table\.'), out)
- assert re.search(re.compile(r'Getting a single greeting by row key.'), out)
- assert re.search(re.compile(r'greeting0: Hello World!'), out)
- assert re.search(re.compile(r'Scanning for all greetings'), out)
- assert re.search(re.compile(r'greeting1: Hello Cloud Bigtable!'), out)
- assert re.search(
- re.compile(r'Deleting the Hello-Bigtable-[0-9]+ table\.'), out)
+ assert 'Creating the {} table.'.format(table_name) in out
+ assert 'Writing some greetings to the table.' in out
+ assert 'Getting a single greeting by row key.' in out
+ assert 'Hello World!' in out
+ assert 'Scanning for all greetings' in out
+ assert 'Hello Cloud Bigtable!' in out
+ assert 'Deleting the {} table.'.format(table_name) in out
diff --git a/nox.py b/nox.py
index d6458265b7b1..f2076c171711 100644
--- a/nox.py
+++ b/nox.py
@@ -30,7 +30,6 @@
"""
import fnmatch
-import itertools
import os
import subprocess
import tempfile
@@ -46,16 +45,6 @@
'-x', '--no-success-flaky-report', '--cov', '--cov-config',
'.coveragerc', '--cov-append', '--cov-report=']
-# Blacklists of samples to ingnore.
-# Bigtable and Speech are disabled because they use gRPC, which does not yet
-# support Python 3. See: https://github.com/grpc/grpc/issues/282
-TESTS_BLACKLIST = set((
- './appengine/standard',
- './bigtable',
- './speech',
- './testing'))
-APPENGINE_BLACKLIST = set()
-
# Libraries that only work on Python 2.7
PY27_ONLY_LIBRARIES = ['mysql-python']
@@ -143,8 +132,8 @@ def setup_appengine(session):
def run_tests_in_sesssion(
- session, interpreter, use_appengine=False, skip_flaky=False,
- changed_only=False, sample_directories=None):
+ session, interpreter, sample_directories, use_appengine=False,
+ skip_flaky=False, changed_only=False):
"""This is the main function for executing tests.
It:
@@ -173,13 +162,6 @@ def run_tests_in_sesssion(
if skip_flaky:
pytest_args.append('-m not slow and not flaky')
- # session.posargs is any leftover arguments from the command line,
- # which allows users to run a particular test instead of all of them.
- if session.posargs:
- sample_directories = session.posargs
- elif sample_directories is None:
- sample_directories = collect_sample_dirs('.', TESTS_BLACKLIST)
-
if changed_only:
changed_files = get_changed_files()
sample_directories = filter_samples(
@@ -204,43 +186,39 @@ def run_tests_in_sesssion(
@nox.parametrize('interpreter', ['python2.7', 'python3.4'])
def session_tests(session, interpreter):
- """Runs tests"""
- run_tests_in_sesssion(session, interpreter)
+ """Runs tests for all non-gae standard samples."""
+ # session.posargs is any leftover arguments from the command line,
+ # which allows users to run a particular test instead of all of them.
+ if session.posargs:
+ sample_directories = session.posargs
+ elif sample_directories is None:
+ sample_directories = collect_sample_dirs(
+ '.', set('./appengine/standard'))
+
+ run_tests_in_sesssion(session, interpreter, sample_directories)
def session_gae(session):
"""Runs test for GAE Standard samples."""
+ sample_directories = collect_sample_dirs('appengine/standard')
run_tests_in_sesssion(
- session, 'python2.7', use_appengine=True,
- sample_directories=collect_sample_dirs(
- 'appengine/standard',
- APPENGINE_BLACKLIST))
-
-
-def session_grpc(session):
- """Runs tests for samples that need grpc."""
- # TODO: Remove this when grpc supports Python 3.
- run_tests_in_sesssion(
- session,
- 'python2.7',
- sample_directories=itertools.chain(
- collect_sample_dirs('speech'),
- collect_sample_dirs('bigtable')))
+ session, 'python2.7', sample_directories, use_appengine=True)
@nox.parametrize('subsession', ['gae', 'tests'])
def session_travis(session, subsession):
"""On travis, just run with python3.4 and don't run slow or flaky tests."""
if subsession == 'tests':
+ sample_directories = collect_sample_dirs(
+ '.', set('./appengine/standard'))
run_tests_in_sesssion(
- session, 'python3.4', skip_flaky=True, changed_only=True)
+ session, 'python3.4', sample_directories, skip_flaky=True,
+ changed_only=True)
else:
+ sample_directories = collect_sample_dirs('appengine/standard')
run_tests_in_sesssion(
- session, 'python2.7', use_appengine=True, skip_flaky=True,
- changed_only=True,
- sample_directories=collect_sample_dirs(
- 'appengine/standard',
- APPENGINE_BLACKLIST))
+ session, 'python2.7', sample_directories, use_appengine=True,
+ skip_flaky=True, changed_only=True)
def session_lint(session):
diff --git a/speech/api/speech_async_grpc_test.py b/speech/api/speech_async_grpc_test.py
index 61070326bfac..4329bba8ff8b 100644
--- a/speech/api/speech_async_grpc_test.py
+++ b/speech/api/speech_async_grpc_test.py
@@ -13,17 +13,12 @@
import argparse
import re
-import sys
import pytest
from speech_async_grpc import _gcs_uri
from speech_async_grpc import main
-@pytest.mark.skipif(
- sys.version_info >= (3, 0),
- reason=("grpc doesn't yet support python3 "
- 'https://github.com/grpc/grpc/issues/282'))
def test_main(cloud_config, capsys):
input_uri = 'gs://{}/speech/audio.flac'.format(cloud_config.storage_bucket)
diff --git a/speech/api/speech_grpc_test.py b/speech/api/speech_grpc_test.py
index 8b0ae9b6f61d..9e9eb7915c7c 100644
--- a/speech/api/speech_grpc_test.py
+++ b/speech/api/speech_grpc_test.py
@@ -12,17 +12,12 @@
# limitations under the License.
import re
-import sys
import pytest
from speech_grpc import _gcs_uri
from speech_grpc import main
-@pytest.mark.skipif(
- sys.version_info >= (3, 0),
- reason=("grpc doesn't yet support python3 "
- 'https://github.com/grpc/grpc/issues/282'))
def test_main(cloud_config, capsys):
input_uri = 'gs://{}/speech/audio.flac'.format(cloud_config.storage_bucket)
diff --git a/speech/api/speech_streaming_test.py b/speech/api/speech_streaming_test.py
index e3b71b88ac99..b81264fca8fe 100644
--- a/speech/api/speech_streaming_test.py
+++ b/speech/api/speech_streaming_test.py
@@ -14,11 +14,8 @@
import contextlib
import io
import re
-import sys
import time
-import pytest
-
import speech_streaming
@@ -57,10 +54,6 @@ def mock_audio_stream(channels, rate, chunk):
return mock_audio_stream
-@pytest.mark.skipif(
- sys.version_info >= (3, 0),
- reason=("grpc doesn't yet support python3 "
- 'https://github.com/grpc/grpc/issues/282'))
def test_main(resource, monkeypatch, capsys):
monkeypatch.setattr(
speech_streaming, 'record_audio',
From a25007b8543157f03944c90a70880cb543d53fdd Mon Sep 17 00:00:00 2001
From: Jon Wayne Parrott <jonwayne@google.com>
Date: Wed, 17 Aug 2016 14:56:21 -0700
Subject: [PATCH 2/4] Unifiy app engine and python2.7 tests.
Change-Id: Ie0df6747050035b2ef5f937951d5ff955073e6d4
---
appengine/standard/conftest.py | 15 +++++++++++++++
nox.py | 24 +++++++++++++-----------
2 files changed, 28 insertions(+), 11 deletions(-)
diff --git a/appengine/standard/conftest.py b/appengine/standard/conftest.py
index 4fd85dcc9067..f8e5c3261e7a 100644
--- a/appengine/standard/conftest.py
+++ b/appengine/standard/conftest.py
@@ -12,6 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import os
+
+import six
+
+
# Import py.test hooks and fixtures for App Engine
from gcp.testing.appengine import (
login,
@@ -25,3 +30,13 @@
(pytest_runtest_call)
(run_tasks)
(testbed)
+
+
+def pytest_ignore_collect(path, config):
+ """Skip App Engine tests in python 3 and if no SDK is available."""
+ if 'appengine/standard' in str(path):
+ if six.PY3:
+ return True
+ if 'GAE_SDK_PATH' not in os.environ:
+ return True
+ return False
diff --git a/nox.py b/nox.py
index f2076c171711..31e38ac20abb 100644
--- a/nox.py
+++ b/nox.py
@@ -121,8 +121,11 @@ def filter_samples(sample_dirs, changed_files):
def setup_appengine(session):
"""Installs the App Engine SDK."""
# Install the app engine sdk and setup import paths.
+ if session.interpreter.startswith('python3'):
+ return
+
gae_root = os.environ.get('GAE_ROOT', tempfile.gettempdir())
- session.env['PYTHONPATH'] = os.path.join(gae_root, 'google_appengine')
+ session.env['GAE_SDK_PATH'] = os.path.join(gae_root, 'google_appengine')
session.run('gcprepotools', 'download-appengine-sdk', gae_root)
# Create a lib directory to prevent the GAE vendor library from
@@ -132,7 +135,7 @@ def setup_appengine(session):
def run_tests_in_sesssion(
- session, interpreter, sample_directories, use_appengine=False,
+ session, interpreter, sample_directories, use_appengine=True,
skip_flaky=False, changed_only=False):
"""This is the main function for executing tests.
@@ -189,13 +192,12 @@ def session_tests(session, interpreter):
"""Runs tests for all non-gae standard samples."""
# session.posargs is any leftover arguments from the command line,
# which allows users to run a particular test instead of all of them.
- if session.posargs:
- sample_directories = session.posargs
- elif sample_directories is None:
- sample_directories = collect_sample_dirs(
- '.', set('./appengine/standard'))
+ sample_directories = session.posargs
+ if not sample_directories:
+ sample_directories = collect_sample_dirs('.')
- run_tests_in_sesssion(session, interpreter, sample_directories)
+ run_tests_in_sesssion(
+ session, interpreter, sample_directories, skip_flaky=True)
def session_gae(session):
@@ -212,12 +214,12 @@ def session_travis(session, subsession):
sample_directories = collect_sample_dirs(
'.', set('./appengine/standard'))
run_tests_in_sesssion(
- session, 'python3.4', sample_directories, skip_flaky=True,
- changed_only=True)
+ session, 'python3.4', sample_directories,
+ skip_flaky=True, changed_only=True)
else:
sample_directories = collect_sample_dirs('appengine/standard')
run_tests_in_sesssion(
- session, 'python2.7', sample_directories, use_appengine=True,
+ session, 'python2.7', sample_directories,
skip_flaky=True, changed_only=True)
From 0952e8944280fe5bae1f4fcb02d37eeb47177017 Mon Sep 17 00:00:00 2001
From: Jon Wayne Parrott <jonwayne@google.com>
Date: Wed, 17 Aug 2016 15:00:57 -0700
Subject: [PATCH 3/4] Don't skip flaky with session tests
Change-Id: I0d3bdf3d6842339d04abc4ee6ddb26b8f44be3e5
---
nox.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/nox.py b/nox.py
index 31e38ac20abb..2c9e53b4fda6 100644
--- a/nox.py
+++ b/nox.py
@@ -197,7 +197,7 @@ def session_tests(session, interpreter):
sample_directories = collect_sample_dirs('.')
run_tests_in_sesssion(
- session, interpreter, sample_directories, skip_flaky=True)
+ session, interpreter, sample_directories)
def session_gae(session):
From b8792d9164f669133032eb26ab78281acb17c9c5 Mon Sep 17 00:00:00 2001
From: Jon Wayne Parrott <jonwayne@google.com>
Date: Thu, 18 Aug 2016 10:10:46 -0700
Subject: [PATCH 4/4] Fix lint issue and review comments
Change-Id: I02a53961b6411247ef06d84dad7b533cb97d89f7
---
appengine/standard/conftest.py | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/appengine/standard/conftest.py b/appengine/standard/conftest.py
index f8e5c3261e7a..8a770af7a43d 100644
--- a/appengine/standard/conftest.py
+++ b/appengine/standard/conftest.py
@@ -14,9 +14,6 @@
import os
-import six
-
-
# Import py.test hooks and fixtures for App Engine
from gcp.testing.appengine import (
login,
@@ -24,6 +21,7 @@
pytest_runtest_call,
run_tasks,
testbed)
+import six
(login)
(pytest_configure)
@@ -33,7 +31,7 @@
def pytest_ignore_collect(path, config):
- """Skip App Engine tests in python 3 and if no SDK is available."""
+ """Skip App Engine tests in python 3 or if no SDK is available."""
if 'appengine/standard' in str(path):
if six.PY3:
return True