From 50609c499ff7cd8f2430de61ef9086e010b5e2dc Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 8 Jun 2016 14:16:02 -0700 Subject: [PATCH] Add Bigtable hello world sample. --- .coveragerc | 1 + bigtable/hello/README.md | 67 +++++++++++++++++++++ bigtable/hello/main.py | 93 ++++++++++++++++++++++++++++++ bigtable/hello/main_test.py | 47 +++++++++++++++ bigtable/hello/requirements.txt | 1 + conftest.py | 4 +- nox.py | 9 ++- scripts/prepare-testing-project.sh | 8 ++- testing/resources/test-env.tmpl.sh | 2 + 9 files changed, 228 insertions(+), 4 deletions(-) create mode 100644 bigtable/hello/README.md create mode 100644 bigtable/hello/main.py create mode 100644 bigtable/hello/main_test.py create mode 100644 bigtable/hello/requirements.txt diff --git a/.coveragerc b/.coveragerc index 5053cd951f23..3e7b8619c0b5 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,7 @@ [run] include = appengine/* + bigtable/* bigquery/* blog/* cloud_logging/* diff --git a/bigtable/hello/README.md b/bigtable/hello/README.md new file mode 100644 index 000000000000..6fc0473a33de --- /dev/null +++ b/bigtable/hello/README.md @@ -0,0 +1,67 @@ +# Cloud Bigtable Hello World + +This is a simple application that demonstrates using the [Google Cloud Client +Library][gcloud-python] to connect to and interact with Cloud Bigtable. + +[gcloud-python]: https://github.com/GoogleCloudPlatform/gcloud-python + + +## Provision a cluster + +Follow the instructions in the [user documentation](https://cloud.google.com/bigtable/docs/creating-cluster) +to create a Google Cloud Platform project and Cloud Bigtable cluster if necessary. +You'll need to reference your project ID, zone and cluster ID to run the application. + + +## Run the application + +First, set your [Google Application Default Credentials](https://developers.google.com/identity/protocols/application-default-credentials) + +Install the dependencies with pip. + +``` +$ pip install -r requirements.txt +``` + +Run the application. Replace the command-line parameters with values for your cluster. + +``` +$ python main.py my-project my-cluster us-central1-c +``` + +You will see output resembling the following: + +``` +Create table Hello-Bigtable-1234 +Write some greetings to the table +Scan for all greetings: + greeting0: Hello World! + greeting1: Hello Cloud Bigtable! + greeting2: Hello HappyBase! +Delete table Hello-Bigtable-1234 +``` + +## Understanding the code + +The [application](main.py) uses the [Google Cloud Bigtable HappyBase +package][Bigtable HappyBase], an implementation of the [HappyBase][HappyBase] +library, to make calls to Cloud Bigtable. It demonstrates several basic +concepts of working with Cloud Bigtable via this API: + +[Bigtable HappyBase]: https://googlecloudplatform.github.io/gcloud-python/stable/happybase-package.html +[HappyBase]: http://happybase.readthedocs.io/en/latest/index.html + +- Creating a [Connection][HappyBase Connection] to a Cloud Bigtable + [Cluster][Cluster API]. +- Using the [Connection][HappyBase Connection] interface to create, disable and + delete a [Table][HappyBase Table]. +- Using the Connection to get a Table. +- Using the Table to write rows via a [put][HappyBase Table Put] and scan + across multiple rows using [scan][HappyBase Table Scan]. + +[Cluster API]: https://googlecloudplatform.github.io/gcloud-python/stable/bigtable-cluster.html +[HappyBase Connection]: https://googlecloudplatform.github.io/gcloud-python/stable/happybase-connection.html +[HappyBase Table]: https://googlecloudplatform.github.io/gcloud-python/stable/happybase-table.html +[HappyBase Table Put]: https://googlecloudplatform.github.io/gcloud-python/stable/happybase-table.html#gcloud.bigtable.happybase.table.Table.put +[HappyBase Table Scan]: https://googlecloudplatform.github.io/gcloud-python/stable/happybase-table.html#gcloud.bigtable.happybase.table.Table.scan + diff --git a/bigtable/hello/main.py b/bigtable/hello/main.py new file mode 100644 index 000000000000..ed19a661a628 --- /dev/null +++ b/bigtable/hello/main.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Demonstrates how to connect to Cloud Bigtable and run some basic operations. + +Prerequisites: + +- Create a Cloud Bigtable cluster. + https://cloud.google.com/bigtable/docs/creating-cluster +- Set your Google Application Default Credentials. + https://developers.google.com/identity/protocols/application-default-credentials +- Set the GCLOUD_PROJECT environment variable to your project ID. + https://support.google.com/cloud/answer/6158840 +""" + +import argparse + +from gcloud import bigtable +from gcloud.bigtable import happybase + + +def main(project, cluster_id, zone, table_name): + column_family_name = 'cf1' + column_name = '{fam}:greeting'.format(fam=column_family_name) + greetings = [ + 'Hello World!', + 'Hello Cloud Bigtable!', + 'Hello HappyBase!', + ] + + # The client must be created with admin=True because it will create a + # table. + client = bigtable.Client(project=project, admin=True) + + with client: + cluster = client.cluster(zone, cluster_id) + cluster.reload() + connection = happybase.Connection(cluster=cluster) + + print('Creating the {} table.'.format(table_name)) + connection.create_table( + table_name, + { + column_family_name: dict() # Use default options. + }) + table = connection.table(table_name) + + print('Writing some greetings to the table.') + for i, value in enumerate(greetings): + row_key = 'greeting{}'.format(i) + table.put(row_key, {column_name: value}) + + print('Scanning for all greetings:') + for key, row in table.scan(): + print('\t{}: {}'.format(key, row[column_name])) + + print('Deleting the {} table.'.format(table_name)) + connection.delete_table(table_name) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='A sample application that connects to Cloud' + + ' Bigtable.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument( + 'project', + help='Google Cloud Platform project ID that contains the Cloud' + + ' Bigtable cluster.') + parser.add_argument( + 'cluster', help='ID of the Cloud Bigtable cluster to connect to.') + parser.add_argument( + 'zone', help='Zone that contains the Cloud Bigtable cluster.') + parser.add_argument( + '--table', + help='Table to create and destroy.', + default='Hello-Bigtable') + + args = parser.parse_args() + main(args.project, args.cluster, args.zone, args.table) diff --git a/bigtable/hello/main_test.py b/bigtable/hello/main_test.py new file mode 100644 index 000000000000..7826a5dbeab3 --- /dev/null +++ b/bigtable/hello/main_test.py @@ -0,0 +1,47 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +import re +import sys + +from hello import main + +import pytest + +TABLE_NAME_FORMAT = 'Hello-Bigtable-{}' +TABLE_NAME_RANGE = 10000 + + +@pytest.mark.skipif( + sys.version_info >= (3, 0), + reason=("grpc doesn't yet support python3 " + 'https://github.com/grpc/grpc/issues/282')) +def test_main(cloud_config, capsys): + table_name = TABLE_NAME_FORMAT.format( + random.randrange(TABLE_NAME_RANGE)) + main( + cloud_config.project, + cloud_config.bigtable_cluster, + cloud_config.bigtable_zone, + table_name) + + out, _ = capsys.readouterr() + assert re.search( + re.compile(r'Creating the Hello-Bigtable-[0-9]+ table\.'), out) + assert re.search(re.compile(r'Writing some greetings to the table\.'), out) + assert re.search(re.compile(r'Scanning for all greetings'), out) + assert re.search(re.compile(r'greeting0: Hello World!'), out) + assert re.search( + re.compile(r'Deleting the Hello-Bigtable-[0-9]+ table\.'), out) diff --git a/bigtable/hello/requirements.txt b/bigtable/hello/requirements.txt new file mode 100644 index 000000000000..93130c943785 --- /dev/null +++ b/bigtable/hello/requirements.txt @@ -0,0 +1 @@ +gcloud[grpc]==0.14.0 diff --git a/conftest.py b/conftest.py index 8bf0e8d2dcf4..1a3348aedbef 100644 --- a/conftest.py +++ b/conftest.py @@ -28,7 +28,9 @@ def cloud_config(): return Namespace( project=os.environ.get('GCLOUD_PROJECT'), storage_bucket=os.environ.get('CLOUD_STORAGE_BUCKET'), - client_secrets=os.environ.get('GOOGLE_CLIENT_SECRETS')) + client_secrets=os.environ.get('GOOGLE_CLIENT_SECRETS'), + bigtable_cluster=os.environ.get('BIGTABLE_CLUSTER'), + bigtable_zone=os.environ.get('BIGTABLE_ZONE')) def get_resource_path(resource, local_path): diff --git a/nox.py b/nox.py index 46f2abd333d0..af042e1ea852 100644 --- a/nox.py +++ b/nox.py @@ -26,8 +26,13 @@ '-x', '--no-success-flaky-report', '--cov', '--cov-config', '.coveragerc', '--cov-append', '--cov-report='] -# Speech is temporarily disabled. -TESTS_BLACKLIST = set(('./appengine/standard', './testing', './speech')) +# Bigtable and Speech are disabled because they use gRPC, which does not yet +# support Python 3. See: https://github.com/grpc/grpc/issues/282 +TESTS_BLACKLIST = set(( + './appengine/standard', + './bigtable', + './speech', + './testing')) APPENGINE_BLACKLIST = set() diff --git a/scripts/prepare-testing-project.sh b/scripts/prepare-testing-project.sh index fafcf89eae1f..c66ddf3e3c67 100755 --- a/scripts/prepare-testing-project.sh +++ b/scripts/prepare-testing-project.sh @@ -22,6 +22,12 @@ echo "Creating cloud storage bucket." gsutil mb gs://$GCLOUD_PROJECT gsutil defacl set public-read gs://$GCLOUD_PROJECT +echo "Creating bigtable resources." +gcloud alpha bigtable clusters create bigtable-test \ + --description="Test cluster" \ + --nodes=3 \ + --zone=us-central1-c + echo "Creating bigquery resources." gcloud alpha bigquery datasets create test_dataset gcloud alpha bigquery datasets create ephemeral_test_dataset @@ -38,4 +44,4 @@ echo "Creating pubsub resources." gcloud alpha pubsub topics create gae-mvm-pubsub-topic echo "To finish setup, follow this link to enable APIs." -echo "https://console.cloud.google.com/flows/enableapi?project=${GCLOUD_PROJECT}&apiid=bigquery,cloudmonitoring,compute_component,datastore,datastore.googleapis.com,dataproc,dns,plus,pubsub,logging,storage_api,vision.googleapis.com" +echo "https://console.cloud.google.com/flows/enableapi?project=${GCLOUD_PROJECT}&apiid=bigtable,bigtableclusteradmin,bigtabletableadmin,bigquery,cloudmonitoring,compute_component,datastore,datastore.googleapis.com,dataproc,dns,plus,pubsub,logging,storage_api,vision.googleapis.com" diff --git a/testing/resources/test-env.tmpl.sh b/testing/resources/test-env.tmpl.sh index e1a03d65cc83..65a53826f8f1 100644 --- a/testing/resources/test-env.tmpl.sh +++ b/testing/resources/test-env.tmpl.sh @@ -1,6 +1,8 @@ # Environment variables for system tests. export GCLOUD_PROJECT=your-project-id export CLOUD_STORAGE_BUCKET=$GCLOUD_PROJECT +export BIGTABLE_CLUSTER=bigtable-test +export BIGTABLE_ZONE=us-central1-c # Environment variables for App Engine Flexible system tests. export GA_TRACKING_ID=