From 0273dd66d267ac90d920673ca21a110207937097 Mon Sep 17 00:00:00 2001 From: Brandon Yarbrough Date: Thu, 23 Mar 2017 16:24:44 -0700 Subject: [PATCH 01/11] Adds storage Pub/Sub notification polling tutorial --- storage/cloud-client/README.rst | 68 +++++++- storage/cloud-client/notification_polling.py | 162 +++++++++++++++++++ storage/cloud-client/requirements.txt | 1 + 3 files changed, 228 insertions(+), 3 deletions(-) create mode 100644 storage/cloud-client/notification_polling.py diff --git a/storage/cloud-client/README.rst b/storage/cloud-client/README.rst index 89f5365446bd..7f998f509418 100644 --- a/storage/cloud-client/README.rst +++ b/storage/cloud-client/README.rst @@ -154,7 +154,7 @@ To run this sample: This application demonstrates how to manage access control lists (acls) in Google Cloud Storage. - + For more information, see the README.md under /storage and the documentation at https://cloud.google.com/storage/docs/encryption. @@ -227,12 +227,71 @@ To run this sample: same key provided when uploading the blob. rotate Performs a key rotation by re-writing an encrypted blob with a new encryption key. - + optional arguments: -h, --help show this help message and exit +Notification Polling ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +To run this sample: + +.. code-block:: bash + + $ python notification_polling.py mysubsription + + usage: notification_polling.py [subscriptionId] + + optional arguments: + -h, --help show this help message and exit + -p, --project=PROJECT specify the subscription's project ID, rather than + using the application's default project ID. + + This application demonstrates how to poll for GCS notifications from a + Cloud Pub/Sub subscription, parse the incoming message, and acknowledge the + successful processing of the message. + + +This application will work with any subscription configured for pull rather than +push notifications. If you do not already have notifications configured, you may +consult the docs at +https://cloud.google.com/storage/docs/reporting-changes or follow the steps +below: + +1. [Activate the Google Cloud Pub/Sub API], if you have not already done so. +2. Create a Google Cloud Storage bucket: + + ``` + $ gsutil mb gs://testbucket + ``` + +3. Create a Cloud Pub/Sub topic and publish bucket notifications there: + + ``` + $ gsutil notification create -f json -t testtopic gs://testbucket + ``` + +4. Create a subscription for your new topic: + + ``` + $ gcloud beta pubsub subscriptions create testsubscription --topic=testtopic + ``` + +5. Run this program: + + ``` + $ python notification_polling testsubscription` + ``` + +6. While the program is running, upload and delete some files in the testbucket + bucket (you could use the console or gsutil) and watch as changes scroll by + in the app. + + The client library ------------------------------------------------------------------------------- @@ -249,4 +308,7 @@ to `browse the source`_ and `report issues`_. https://github.com/GoogleCloudPlatform/google-cloud-python/issues -.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file +.. _Google Cloud SDK: https://cloud.google.com/sdk/ + + +[Activate the Google Cloud Pub/Sub API]: https://console.cloud.google.com/flows/enableapi?apiid=storage_component diff --git a/storage/cloud-client/notification_polling.py b/storage/cloud-client/notification_polling.py new file mode 100644 index 000000000000..a70ec1974359 --- /dev/null +++ b/storage/cloud-client/notification_polling.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to poll a Google Cloud Pub/Sub +subscription for notification messages, parse those messages, and +acknowledge successful handling of them. + +For more information, see the README.md file in this directory or the +docs at https://cloud.google.com/storage/docs/pubsub-notifications + +Example usage: + python notification_poller.py bucketsubscription +""" + +from __future__ import print_function + +import argparse +import json +import sys + +from cachetools import LRUCache +from google.cloud import pubsub + + +ACTION_STRINGS = { + 'OBJECT_ARCHIVE': 'Object archived', + 'OBJECT_DELETE': 'Object deleted', + 'OBJECT_FINALIZE': 'Object created', + 'OBJECT_METADATA_UPDATE': 'Object metadata updated' +} + + +DUPLICATE_EVENT_CACHE = LRUCache(10000) +DUPLICATION_KEY = '{event_type}|{resource}|{generation}|{metageneration}' + + +class GcsEvent(object): + """Represents a single event message from GCS.""" + + def __init__(self, message): + """Initializes a GcsEvent with a Pub/Sub message.""" + # [START parse_message] + data = message.data + attributes = message.attributes + + # The kind of event that just happened. Example: OBJECT_FINALIZE + self.event_type = attributes['eventType'] + + # ID of the bucket. Example: "my_photos" + self.bucket_id = attributes['bucketId'] + + # The ID of the affected objet. Example: "image.jpeg" + self.object_id = attributes['objectId'] + + # Generation of the object. Example: 1234567 + self.generation = attributes['objectGeneration'] + + # The full resource name of the object. + # Example: projects/_/buckets/my_photos/objects/image.jpeg + self.object_name = attributes['resource'] + + # Format of the attached payload. Example: JSON_API_V1 + self.payload_format = attributes['payloadFormat'] + + # ID of the notification configuration responsible for this event. + # Example: projects/_/buckets/my_photos/notificationConfigs/1 + self.notification_config = attributes['notificationConfig'] + + if self.payload_format == 'JSON_API_V1': + # The payload is the JSON API object resource. + self.object_metadata = json.loads(data) + self.object_size = self.object_metadata['size'] + self.content_type = self.object_metadata['contentType'] + self.metageneration = self.object_metadata['metageneration'] + elif self.payloadFormat == 'NONE': + # There is no payload. + self.object_metadata = None + self.metageneration = None + else: + print('Unknown payload format %s', self.payload_format) + self.object_metadata = None + self.metageneration = None + # [END parse_message] + + def Summary(self): + """Returns a one line summary of the event.""" + return '%s - %s/%s' % ( + ACTION_STRINGS.get( + self.event_type, 'Unknown event %s' % self.event_type), + self.bucket_id, + self.object_id) + + def __unicode__(self): + description = ( + '{summary}\n' + '\tGeneration: {generation}\n').format( + summary=self.Summary(), + bucket_id=self.bucket_id, + object_id=self.object_id, + generation=self.generation) + if self.object_metadata: + description += ( + '\tContent type: {content_type}\n' + '\tSize: {object_size}\n').format( + content_type=self.content_type, + object_size=self.object_size) + return description + + def dup_string(self): + """Returns a string unique to this specific event.""" + # GCS will publish each notification at least once to a Cloud Pub/Sub + # topic, and Cloud Pub/Sub will deliver each message to each subscription + # at least once. We must be able to safely handle occasionally receiving + # duplicate messages. + return DUPLICATION_KEY.format( + event_type=self.event_type, + resource=self.object_name, + generation=self.generation, + metageneration=self.metageneration) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__) + parser.add_argument( + '--project', + help='The project of the subscription, if not in your default project') + parser.add_argument('subscription', help='The ID of the Pub/Sub subscription') + args = parser.parse_args() + + subscription_id = args.subscription + client = pubsub.Client(project=args.project) + subscription = pubsub.subscription.Subscription( + subscription_id, client=client) + if not subscription.exists(): + sys.stderr.write('Cannot find subscription %s\n' % sys.argv[1]) + sys.exit(1) + + print('Polling for messages.') + while True: + pulled = subscription.pull(max_messages=100) + for ack_id, message in pulled: + event = GcsEvent(message) + if event.dup_string() in DUPLICATE_EVENT_CACHE: + print('[DUPLICATE] %s' % event.Summary()) + else: + DUPLICATE_EVENT_CACHE[event.dup_string()] = 1 + print(unicode(event)) + subscription.acknowledge([ack_id]) diff --git a/storage/cloud-client/requirements.txt b/storage/cloud-client/requirements.txt index a12f0fc3054b..a29c865f4497 100644 --- a/storage/cloud-client/requirements.txt +++ b/storage/cloud-client/requirements.txt @@ -1 +1,2 @@ google-cloud-storage==0.22.0 +cachetools>=2.0.0 From b81aa1fe6a89deefb8d6e72e5505abcfba5501a3 Mon Sep 17 00:00:00 2001 From: Brandon Yarbrough Date: Thu, 23 Mar 2017 17:22:41 -0700 Subject: [PATCH 02/11] Fix formatting and add some tests --- storage/cloud-client/notification_polling.py | 219 +++++++++--------- .../cloud-client/notification_polling_test.py | 62 +++++ 2 files changed, 172 insertions(+), 109 deletions(-) create mode 100644 storage/cloud-client/notification_polling_test.py diff --git a/storage/cloud-client/notification_polling.py b/storage/cloud-client/notification_polling.py index a70ec1974359..f30bb79348ab 100644 --- a/storage/cloud-client/notification_polling.py +++ b/storage/cloud-client/notification_polling.py @@ -48,115 +48,116 @@ class GcsEvent(object): - """Represents a single event message from GCS.""" - - def __init__(self, message): - """Initializes a GcsEvent with a Pub/Sub message.""" - # [START parse_message] - data = message.data - attributes = message.attributes - - # The kind of event that just happened. Example: OBJECT_FINALIZE - self.event_type = attributes['eventType'] - - # ID of the bucket. Example: "my_photos" - self.bucket_id = attributes['bucketId'] - - # The ID of the affected objet. Example: "image.jpeg" - self.object_id = attributes['objectId'] - - # Generation of the object. Example: 1234567 - self.generation = attributes['objectGeneration'] - - # The full resource name of the object. - # Example: projects/_/buckets/my_photos/objects/image.jpeg - self.object_name = attributes['resource'] - - # Format of the attached payload. Example: JSON_API_V1 - self.payload_format = attributes['payloadFormat'] - - # ID of the notification configuration responsible for this event. - # Example: projects/_/buckets/my_photos/notificationConfigs/1 - self.notification_config = attributes['notificationConfig'] - - if self.payload_format == 'JSON_API_V1': - # The payload is the JSON API object resource. - self.object_metadata = json.loads(data) - self.object_size = self.object_metadata['size'] - self.content_type = self.object_metadata['contentType'] - self.metageneration = self.object_metadata['metageneration'] - elif self.payloadFormat == 'NONE': - # There is no payload. - self.object_metadata = None - self.metageneration = None - else: - print('Unknown payload format %s', self.payload_format) - self.object_metadata = None - self.metageneration = None - # [END parse_message] - - def Summary(self): - """Returns a one line summary of the event.""" - return '%s - %s/%s' % ( - ACTION_STRINGS.get( - self.event_type, 'Unknown event %s' % self.event_type), - self.bucket_id, - self.object_id) - - def __unicode__(self): - description = ( - '{summary}\n' - '\tGeneration: {generation}\n').format( - summary=self.Summary(), - bucket_id=self.bucket_id, - object_id=self.object_id, - generation=self.generation) - if self.object_metadata: - description += ( - '\tContent type: {content_type}\n' - '\tSize: {object_size}\n').format( - content_type=self.content_type, - object_size=self.object_size) - return description - - def dup_string(self): - """Returns a string unique to this specific event.""" - # GCS will publish each notification at least once to a Cloud Pub/Sub - # topic, and Cloud Pub/Sub will deliver each message to each subscription - # at least once. We must be able to safely handle occasionally receiving - # duplicate messages. - return DUPLICATION_KEY.format( - event_type=self.event_type, - resource=self.object_name, - generation=self.generation, - metageneration=self.metageneration) + """Represents a single event message from GCS.""" + + def __init__(self, message): + """Initializes a GcsEvent with a Pub/Sub message.""" + # [START parse_message] + data = message.data + attributes = message.attributes + + # The kind of event that just happened. Example: OBJECT_FINALIZE + self.event_type = attributes['eventType'] + + # ID of the bucket. Example: "my_photos" + self.bucket_id = attributes['bucketId'] + + # The ID of the affected objet. Example: "image.jpeg" + self.object_id = attributes['objectId'] + + # Generation of the object. Example: 1234567 + self.generation = attributes['objectGeneration'] + + # The full resource name of the object. + # Example: projects/_/buckets/my_photos/objects/image.jpeg + self.object_name = attributes['resource'] + + # Format of the attached payload. Example: JSON_API_V1 + self.payload_format = attributes['payloadFormat'] + + # ID of the notification configuration responsible for this event. + # Example: projects/_/buckets/my_photos/notificationConfigs/1 + self.notification_config = attributes['notificationConfig'] + + if self.payload_format == 'JSON_API_V1': + # The payload is the JSON API object resource. + self.object_metadata = json.loads(data) + self.object_size = self.object_metadata['size'] + self.content_type = self.object_metadata['contentType'] + self.metageneration = self.object_metadata['metageneration'] + elif self.payload_format == 'NONE': + # There is no payload. + self.object_metadata = None + self.metageneration = None + else: + print('Unknown payload format %s', self.payload_format) + self.object_metadata = None + self.metageneration = None + # [END parse_message] + + def Summary(self): + """Returns a one line summary of the event.""" + return '%s - %s/%s' % ( + ACTION_STRINGS.get( + self.event_type, 'Unknown event %s' % self.event_type), + self.bucket_id, + self.object_id) + + def __unicode__(self): + description = ( + '{summary}\n' + '\tGeneration: {generation}\n').format( + summary=self.Summary(), + bucket_id=self.bucket_id, + object_id=self.object_id, + generation=self.generation) + if self.object_metadata: + description += ( + '\tContent type: {content_type}\n' + '\tSize: {object_size}\n').format( + content_type=self.content_type, + object_size=self.object_size) + return description + + def dup_string(self): + """Returns a string unique to this specific event.""" + # GCS will publish each notification at least once to a Cloud Pub/Sub + # topic, and Cloud Pub/Sub will deliver each message to each + # subscription at least once. We must be able to safely handle + # occasionally receiving duplicate messages. + return DUPLICATION_KEY.format( + event_type=self.event_type, + resource=self.object_name, + generation=self.generation, + metageneration=self.metageneration) if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__) - parser.add_argument( - '--project', - help='The project of the subscription, if not in your default project') - parser.add_argument('subscription', help='The ID of the Pub/Sub subscription') - args = parser.parse_args() - - subscription_id = args.subscription - client = pubsub.Client(project=args.project) - subscription = pubsub.subscription.Subscription( - subscription_id, client=client) - if not subscription.exists(): - sys.stderr.write('Cannot find subscription %s\n' % sys.argv[1]) - sys.exit(1) - - print('Polling for messages.') - while True: - pulled = subscription.pull(max_messages=100) - for ack_id, message in pulled: - event = GcsEvent(message) - if event.dup_string() in DUPLICATE_EVENT_CACHE: - print('[DUPLICATE] %s' % event.Summary()) - else: - DUPLICATE_EVENT_CACHE[event.dup_string()] = 1 - print(unicode(event)) - subscription.acknowledge([ack_id]) + parser = argparse.ArgumentParser( + description=__doc__) + parser.add_argument( + '--project', + help='The project of the subscription, if not in your default project') + parser.add_argument('subscription', + help='The ID of the Pub/Sub subscription') + args = parser.parse_args() + + subscription_id = args.subscription + client = pubsub.Client(project=args.project) + subscription = pubsub.subscription.Subscription( + subscription_id, client=client) + if not subscription.exists(): + sys.stderr.write('Cannot find subscription %s\n' % sys.argv[1]) + sys.exit(1) + + print('Polling for messages.') + while True: + pulled = subscription.pull(max_messages=100) + for ack_id, message in pulled: + event = GcsEvent(message) + if event.dup_string() in DUPLICATE_EVENT_CACHE: + print('[DUPLICATE] %s' % event.Summary()) + else: + DUPLICATE_EVENT_CACHE[event.dup_string()] = 1 + print(unicode(event)) + subscription.acknowledge([ack_id]) diff --git a/storage/cloud-client/notification_polling_test.py b/storage/cloud-client/notification_polling_test.py new file mode 100644 index 000000000000..935947f38637 --- /dev/null +++ b/storage/cloud-client/notification_polling_test.py @@ -0,0 +1,62 @@ +# Copyright 2017 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from google.cloud.pubsub.message import Message +from notification_polling import GcsEvent + + +MESSAGE_ID = 12345 + + +def test_parse_json_message(): + attributes = { + 'eventType': 'OBJECT_FINALIZE', + 'bucketId': 'mybucket', + 'objectId': 'myobject', + 'objectGeneration': 1234567, + 'resource': 'projects/_/buckets/mybucket/objects/myobject#1234567', + 'notificationConfig': ('projects/_/buckets/mybucket/' + 'notificationConfigs/5'), + 'payloadFormat': 'JSON_API_V1'} + data = ('{' + ' "size": 12345,' + ' "contentType": "text/html",' + ' "metageneration": 1' + '}') + message = Message(data, MESSAGE_ID, attributes=attributes) + event = GcsEvent(message) + assert unicode(event) == (u'Object created - mybucket/myobject\n' + '\tGeneration: 1234567\n' + '\tContent type: text/html\n' + '\tSize: 12345\n') + assert event.Summary() == 'Object created - mybucket/myobject' + + +def test_parse_no_payload_message(): + attributes = { + 'eventType': 'OBJECT_FINALIZE', + 'bucketId': 'mybucket', + 'objectId': 'myobject', + 'objectGeneration': 1234567, + 'resource': 'projects/_/buckets/mybucket/objects/myobject#1234567', + 'notificationConfig': ('projects/_/buckets/mybucket/' + 'notificationConfigs/5'), + 'payloadFormat': 'NONE'} + data = None + message = Message(data, MESSAGE_ID, attributes=attributes) + event = GcsEvent(message) + assert unicode(event) == (u'Object created - mybucket/myobject\n' + '\tGeneration: 1234567\n') + assert event.Summary() == 'Object created - mybucket/myobject' From f31c22fcd1349afe0e4100096204674de2275a22 Mon Sep 17 00:00:00 2001 From: Brandon Yarbrough Date: Thu, 23 Mar 2017 17:46:53 -0700 Subject: [PATCH 03/11] Auto-generate README --- storage/cloud-client/README.rst | 81 +++++++------------ storage/cloud-client/README.rst.in | 3 + storage/cloud-client/notification_polling.py | 45 ++++++++--- .../cloud-client/notification_polling_test.py | 1 + storage/cloud-client/requirements.txt | 1 + 5 files changed, 70 insertions(+), 61 deletions(-) diff --git a/storage/cloud-client/README.rst b/storage/cloud-client/README.rst index 7f998f509418..0089346172b5 100644 --- a/storage/cloud-client/README.rst +++ b/storage/cloud-client/README.rst @@ -154,7 +154,7 @@ To run this sample: This application demonstrates how to manage access control lists (acls) in Google Cloud Storage. - + For more information, see the README.md under /storage and the documentation at https://cloud.google.com/storage/docs/encryption. @@ -227,12 +227,11 @@ To run this sample: same key provided when uploading the blob. rotate Performs a key rotation by re-writing an encrypted blob with a new encryption key. - + optional arguments: -h, --help show this help message and exit - Notification Polling +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -242,54 +241,35 @@ To run this sample: .. code-block:: bash - $ python notification_polling.py mysubsription - - usage: notification_polling.py [subscriptionId] + $ python notification_polling.py + usage: notification_polling.py [-h] [--project PROJECT] subscription + + This application demonstrates how to poll for GCS notifications from a Cloud + Pub/Sub subscription, parse the incoming message, and acknowledge the + successful processing of the message. This application will work with any + subscription configured for pull rather than push notifications. If you do not + already have notifications configured, you may consult the docs at + https://cloud.google.com/storage/docs/reporting-changes or follow the steps + below: 1. [Activate the Google Cloud Pub/Sub API], if you have not already + done so. 2. Create a Google Cloud Storage bucket: ``` $ gsutil mb + gs://testbucket ``` 3. Create a Cloud Pub/Sub topic and publish bucket + notifications there: ``` $ gsutil notification create -f json -t testtopic + gs://testbucket ``` 4. Create a subscription for your new topic: ``` $ gcloud + beta pubsub subscriptions create testsubscription --topic=testtopic ``` 5. Run + this program: ``` $ python notification_polling testsubscription` ``` 6. While + the program is running, upload and delete some files in the testbucket bucket + (you could use the console or gsutil) and watch as changes scroll by in the + app. + + positional arguments: + subscription The ID of the Pub/Sub subscription + optional arguments: - -h, --help show this help message and exit - -p, --project=PROJECT specify the subscription's project ID, rather than - using the application's default project ID. - - This application demonstrates how to poll for GCS notifications from a - Cloud Pub/Sub subscription, parse the incoming message, and acknowledge the - successful processing of the message. - + -h, --help show this help message and exit + --project PROJECT The project of the subscription, if not in your default + project -This application will work with any subscription configured for pull rather than -push notifications. If you do not already have notifications configured, you may -consult the docs at -https://cloud.google.com/storage/docs/reporting-changes or follow the steps -below: - -1. [Activate the Google Cloud Pub/Sub API], if you have not already done so. -2. Create a Google Cloud Storage bucket: - - ``` - $ gsutil mb gs://testbucket - ``` - -3. Create a Cloud Pub/Sub topic and publish bucket notifications there: - - ``` - $ gsutil notification create -f json -t testtopic gs://testbucket - ``` - -4. Create a subscription for your new topic: - - ``` - $ gcloud beta pubsub subscriptions create testsubscription --topic=testtopic - ``` - -5. Run this program: - - ``` - $ python notification_polling testsubscription` - ``` - -6. While the program is running, upload and delete some files in the testbucket - bucket (you could use the console or gsutil) and watch as changes scroll by - in the app. @@ -308,7 +288,4 @@ to `browse the source`_ and `report issues`_. https://github.com/GoogleCloudPlatform/google-cloud-python/issues -.. _Google Cloud SDK: https://cloud.google.com/sdk/ - - -[Activate the Google Cloud Pub/Sub API]: https://console.cloud.google.com/flows/enableapi?apiid=storage_component +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/storage/cloud-client/README.rst.in b/storage/cloud-client/README.rst.in index 2a6e37ff7745..aa9690abe13f 100644 --- a/storage/cloud-client/README.rst.in +++ b/storage/cloud-client/README.rst.in @@ -24,5 +24,8 @@ samples: - name: Customer-Supplied Encryption file: encryption.py show_help: true +- name: Notification Polling + file: notification_polling.py + show_help: true cloud_client_library: true diff --git a/storage/cloud-client/notification_polling.py b/storage/cloud-client/notification_polling.py index f30bb79348ab..b44c7613c652 100644 --- a/storage/cloud-client/notification_polling.py +++ b/storage/cloud-client/notification_polling.py @@ -14,18 +14,45 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""This application demonstrates how to poll a Google Cloud Pub/Sub -subscription for notification messages, parse those messages, and -acknowledge successful handling of them. +"""This application demonstrates how to poll for GCS notifications from a +Cloud Pub/Sub subscription, parse the incoming message, and acknowledge the +successful processing of the message. -For more information, see the README.md file in this directory or the -docs at https://cloud.google.com/storage/docs/pubsub-notifications +This application will work with any subscription configured for pull rather +than push notifications. If you do not already have notifications configured, +you may consult the docs at +https://cloud.google.com/storage/docs/reporting-changes or follow the steps +below: -Example usage: - python notification_poller.py bucketsubscription -""" +1. [Activate the Google Cloud Pub/Sub API], if you have not already done so. +2. Create a Google Cloud Storage bucket: + + ``` + $ gsutil mb gs://testbucket + ``` + +3. Create a Cloud Pub/Sub topic and publish bucket notifications there: + + ``` + $ gsutil notification create -f json -t testtopic gs://testbucket + ``` + +4. Create a subscription for your new topic: -from __future__ import print_function + ``` + $ gcloud beta pubsub subscriptions create testsubscription --topic=testtopic + ``` + +5. Run this program: + + ``` + $ python notification_polling testsubscription` + ``` + +6. While the program is running, upload and delete some files in the testbucket + bucket (you could use the console or gsutil) and watch as changes scroll by + in the app. +""" import argparse import json diff --git a/storage/cloud-client/notification_polling_test.py b/storage/cloud-client/notification_polling_test.py index 935947f38637..e25587050ca5 100644 --- a/storage/cloud-client/notification_polling_test.py +++ b/storage/cloud-client/notification_polling_test.py @@ -14,6 +14,7 @@ from google.cloud.pubsub.message import Message + from notification_polling import GcsEvent diff --git a/storage/cloud-client/requirements.txt b/storage/cloud-client/requirements.txt index a29c865f4497..a93b508e2007 100644 --- a/storage/cloud-client/requirements.txt +++ b/storage/cloud-client/requirements.txt @@ -1,2 +1,3 @@ google-cloud-storage==0.22.0 +google-cloud-pubsub==0.22.0 cachetools>=2.0.0 From 5bb12737492f0a46dd58c35d90b8c1eb6463627d Mon Sep 17 00:00:00 2001 From: Brandon Yarbrough Date: Fri, 24 Mar 2017 09:46:53 -0700 Subject: [PATCH 04/11] Simplify implementation, remove classes --- storage/cloud-client/notification_polling.py | 161 ++++++++---------- .../cloud-client/notification_polling_test.py | 26 +-- 2 files changed, 85 insertions(+), 102 deletions(-) diff --git a/storage/cloud-client/notification_polling.py b/storage/cloud-client/notification_polling.py index b44c7613c652..fc54c4bbdf86 100644 --- a/storage/cloud-client/notification_polling.py +++ b/storage/cloud-client/notification_polling.py @@ -71,92 +71,70 @@ DUPLICATE_EVENT_CACHE = LRUCache(10000) -DUPLICATION_KEY = '{event_type}|{resource}|{generation}|{metageneration}' - - -class GcsEvent(object): - """Represents a single event message from GCS.""" - - def __init__(self, message): - """Initializes a GcsEvent with a Pub/Sub message.""" - # [START parse_message] - data = message.data - attributes = message.attributes - - # The kind of event that just happened. Example: OBJECT_FINALIZE - self.event_type = attributes['eventType'] - - # ID of the bucket. Example: "my_photos" - self.bucket_id = attributes['bucketId'] - - # The ID of the affected objet. Example: "image.jpeg" - self.object_id = attributes['objectId'] - - # Generation of the object. Example: 1234567 - self.generation = attributes['objectGeneration'] - - # The full resource name of the object. - # Example: projects/_/buckets/my_photos/objects/image.jpeg - self.object_name = attributes['resource'] - - # Format of the attached payload. Example: JSON_API_V1 - self.payload_format = attributes['payloadFormat'] - - # ID of the notification configuration responsible for this event. - # Example: projects/_/buckets/my_photos/notificationConfigs/1 - self.notification_config = attributes['notificationConfig'] - - if self.payload_format == 'JSON_API_V1': - # The payload is the JSON API object resource. - self.object_metadata = json.loads(data) - self.object_size = self.object_metadata['size'] - self.content_type = self.object_metadata['contentType'] - self.metageneration = self.object_metadata['metageneration'] - elif self.payload_format == 'NONE': - # There is no payload. - self.object_metadata = None - self.metageneration = None - else: - print('Unknown payload format %s', self.payload_format) - self.object_metadata = None - self.metageneration = None - # [END parse_message] - - def Summary(self): - """Returns a one line summary of the event.""" - return '%s - %s/%s' % ( - ACTION_STRINGS.get( - self.event_type, 'Unknown event %s' % self.event_type), - self.bucket_id, - self.object_id) - - def __unicode__(self): - description = ( - '{summary}\n' - '\tGeneration: {generation}\n').format( - summary=self.Summary(), - bucket_id=self.bucket_id, - object_id=self.object_id, - generation=self.generation) - if self.object_metadata: - description += ( - '\tContent type: {content_type}\n' - '\tSize: {object_size}\n').format( - content_type=self.content_type, - object_size=self.object_size) - return description - - def dup_string(self): - """Returns a string unique to this specific event.""" - # GCS will publish each notification at least once to a Cloud Pub/Sub - # topic, and Cloud Pub/Sub will deliver each message to each - # subscription at least once. We must be able to safely handle - # occasionally receiving duplicate messages. - return DUPLICATION_KEY.format( - event_type=self.event_type, - resource=self.object_name, - generation=self.generation, - metageneration=self.metageneration) +DUPLICATION_KEY = '{event_type}|{resource}|{metageneration}' + + +def Summarize(message): + # [START parse_message] + data = message.data + attributes = message.attributes + + # The kind of event that just happened. Example: OBJECT_FINALIZE + event_type = attributes['eventType'] + event_description = ACTION_STRINGS.get( + event_type, 'Unknown Event %s' % event_type) + + # ID of the bucket. Example: "my_photos" + bucket_id = attributes['bucketId'] + # The ID of the affected objet. Example: "image.jpeg" + object_id = attributes['objectId'] + # Generation of the object. Example: 1234567 + generation = attributes['objectGeneration'] + # Format of the attached payload. Example: JSON_API_V1 + payload_format = attributes['payloadFormat'] + description = ( + '{summary} - {bucket_id}/{object_id}\n' + '\tGeneration: {generation}\n').format( + summary=event_description, + bucket_id=bucket_id, + object_id=object_id, + generation=generation) + + if payload_format == 'JSON_API_V1': + # The payload is the JSON API object resource. + object_metadata = json.loads(data) + size = object_metadata['size'] + content_type = object_metadata['contentType'] + metageneration = object_metadata['metageneration'] + description += ( + '\tContent type: {content_type}\n' + '\tSize: {object_size}\n' + '\tMetageneration: {metageneration}\n').format( + content_type=content_type, + object_size=size, + metageneration=metageneration) + elif payload_format == 'NONE': + # There is no payload. + pass + else: + print('Unknown payload format %s', payload_format) + return description + # [END parse_message] + + +def GetDedupString(message): + """Returns a string unique to this specific event.""" + # GCS will publish each notification at least once to a Cloud Pub/Sub + # topic, and Cloud Pub/Sub will deliver each message to each + # subscription at least once. We must be able to safely handle + # occasionally receiving duplicate messages. + metageneration = 'unknown' + if message.attributes['payloadFormat'] == 'JSON_API_V1': + metageneration = json.loads(message.data)['metageneration'] + return DUPLICATION_KEY.format( + event_type=message.attributes['eventType'], + resource=message.attributes['resource'], + metageneration=metageneration) if __name__ == '__main__': @@ -181,10 +159,11 @@ def dup_string(self): while True: pulled = subscription.pull(max_messages=100) for ack_id, message in pulled: - event = GcsEvent(message) - if event.dup_string() in DUPLICATE_EVENT_CACHE: - print('[DUPLICATE] %s' % event.Summary()) + dup_string = GetDedupString(message) + summary = Summarize(message) + if dup_string in DUPLICATE_EVENT_CACHE: + print('[DUPLICATE] %s' % summary) else: - DUPLICATE_EVENT_CACHE[event.dup_string()] = 1 - print(unicode(event)) + DUPLICATE_EVENT_CACHE[dup_string] = 1 + print(summary) subscription.acknowledge([ack_id]) diff --git a/storage/cloud-client/notification_polling_test.py b/storage/cloud-client/notification_polling_test.py index e25587050ca5..ffc4aa5fd156 100644 --- a/storage/cloud-client/notification_polling_test.py +++ b/storage/cloud-client/notification_polling_test.py @@ -15,7 +15,8 @@ from google.cloud.pubsub.message import Message -from notification_polling import GcsEvent +from notification_polling import GetDedupString +from notification_polling import Summarize MESSAGE_ID = 12345 @@ -37,12 +38,14 @@ def test_parse_json_message(): ' "metageneration": 1' '}') message = Message(data, MESSAGE_ID, attributes=attributes) - event = GcsEvent(message) - assert unicode(event) == (u'Object created - mybucket/myobject\n' - '\tGeneration: 1234567\n' - '\tContent type: text/html\n' - '\tSize: 12345\n') - assert event.Summary() == 'Object created - mybucket/myobject' + assert Summarize(message) == ('Object created - mybucket/myobject\n' + '\tGeneration: 1234567\n' + '\tContent type: text/html\n' + '\tSize: 12345\n' + '\tMetageneration: 1\n') + assert GetDedupString(message) == ( + 'OBJECT_FINALIZE|projects/_/buckets/mybucket/' + 'objects/myobject#1234567|1') def test_parse_no_payload_message(): @@ -57,7 +60,8 @@ def test_parse_no_payload_message(): 'payloadFormat': 'NONE'} data = None message = Message(data, MESSAGE_ID, attributes=attributes) - event = GcsEvent(message) - assert unicode(event) == (u'Object created - mybucket/myobject\n' - '\tGeneration: 1234567\n') - assert event.Summary() == 'Object created - mybucket/myobject' + assert Summarize(message) == ('Object created - mybucket/myobject\n' + '\tGeneration: 1234567\n') + assert GetDedupString(message) == ( + 'OBJECT_FINALIZE|projects/_/buckets/mybucket/' + 'objects/myobject#1234567|unknown') From 7a0f11fd5b297ff3cb4543d2945d762db32e870c Mon Sep 17 00:00:00 2001 From: Brandon Yarbrough Date: Fri, 24 Mar 2017 11:07:17 -0700 Subject: [PATCH 05/11] Simplified example, removed de-duping --- storage/cloud-client/notification_polling.py | 110 +++++------------- .../cloud-client/notification_polling_test.py | 38 ++---- storage/cloud-client/requirements.txt | 1 - 3 files changed, 39 insertions(+), 110 deletions(-) diff --git a/storage/cloud-client/notification_polling.py b/storage/cloud-client/notification_polling.py index fc54c4bbdf86..9e90f50cea7f 100644 --- a/storage/cloud-client/notification_polling.py +++ b/storage/cloud-client/notification_polling.py @@ -24,30 +24,20 @@ https://cloud.google.com/storage/docs/reporting-changes or follow the steps below: -1. [Activate the Google Cloud Pub/Sub API], if you have not already done so. -2. Create a Google Cloud Storage bucket: +1. Activate the Google Cloud Pub/Sub API, if you have not already done so. + https://console.cloud.google.com/flows/enableapi?apiid=pubsub - ``` - $ gsutil mb gs://testbucket - ``` +2. Create a Google Cloud Storage bucket: + $ gsutil mb gs://testbucket 3. Create a Cloud Pub/Sub topic and publish bucket notifications there: - - ``` $ gsutil notification create -f json -t testtopic gs://testbucket - ``` 4. Create a subscription for your new topic: - - ``` $ gcloud beta pubsub subscriptions create testsubscription --topic=testtopic - ``` 5. Run this program: - - ``` - $ python notification_polling testsubscription` - ``` + $ python notification_polling testsubscription 6. While the program is running, upload and delete some files in the testbucket bucket (you could use the console or gsutil) and watch as changes scroll by @@ -58,50 +48,30 @@ import json import sys -from cachetools import LRUCache from google.cloud import pubsub -ACTION_STRINGS = { - 'OBJECT_ARCHIVE': 'Object archived', - 'OBJECT_DELETE': 'Object deleted', - 'OBJECT_FINALIZE': 'Object created', - 'OBJECT_METADATA_UPDATE': 'Object metadata updated' -} - - -DUPLICATE_EVENT_CACHE = LRUCache(10000) -DUPLICATION_KEY = '{event_type}|{resource}|{metageneration}' - - -def Summarize(message): +def summarize(message): # [START parse_message] data = message.data attributes = message.attributes - # The kind of event that just happened. Example: OBJECT_FINALIZE event_type = attributes['eventType'] - event_description = ACTION_STRINGS.get( - event_type, 'Unknown Event %s' % event_type) - - # ID of the bucket. Example: "my_photos" bucket_id = attributes['bucketId'] - # The ID of the affected objet. Example: "image.jpeg" object_id = attributes['objectId'] - # Generation of the object. Example: 1234567 generation = attributes['objectGeneration'] - # Format of the attached payload. Example: JSON_API_V1 - payload_format = attributes['payloadFormat'] description = ( - '{summary} - {bucket_id}/{object_id}\n' + '\tEvent type: {event_type}\n' + '\tBucket ID: {bucket_id}\n' + '\tObject ID: {object_id}\n' '\tGeneration: {generation}\n').format( - summary=event_description, + event_type=event_type, bucket_id=bucket_id, object_id=object_id, generation=generation) + payload_format = attributes['payloadFormat'] if payload_format == 'JSON_API_V1': - # The payload is the JSON API object resource. object_metadata = json.loads(data) size = object_metadata['size'] content_type = object_metadata['contentType'] @@ -113,57 +83,37 @@ def Summarize(message): content_type=content_type, object_size=size, metageneration=metageneration) - elif payload_format == 'NONE': - # There is no payload. - pass - else: - print('Unknown payload format %s', payload_format) return description # [END parse_message] -def GetDedupString(message): - """Returns a string unique to this specific event.""" - # GCS will publish each notification at least once to a Cloud Pub/Sub - # topic, and Cloud Pub/Sub will deliver each message to each - # subscription at least once. We must be able to safely handle - # occasionally receiving duplicate messages. - metageneration = 'unknown' - if message.attributes['payloadFormat'] == 'JSON_API_V1': - metageneration = json.loads(message.data)['metageneration'] - return DUPLICATION_KEY.format( - event_type=message.attributes['eventType'], - resource=message.attributes['resource'], - metageneration=metageneration) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__) - parser.add_argument( - '--project', - help='The project of the subscription, if not in your default project') - parser.add_argument('subscription', - help='The ID of the Pub/Sub subscription') - args = parser.parse_args() - - subscription_id = args.subscription +def poll_notifications(project, subscription_id): + """Polls a Cloud Pub/Sub subscription for new GCS events for display.""" + # [BEGIN poll_notifications] client = pubsub.Client(project=args.project) subscription = pubsub.subscription.Subscription( subscription_id, client=client) if not subscription.exists(): - sys.stderr.write('Cannot find subscription %s\n' % sys.argv[1]) + sys.stderr.write('Cannot find subscription {0}\n'.format(sys.argv[1])) sys.exit(1) print('Polling for messages.') while True: pulled = subscription.pull(max_messages=100) for ack_id, message in pulled: - dup_string = GetDedupString(message) - summary = Summarize(message) - if dup_string in DUPLICATE_EVENT_CACHE: - print('[DUPLICATE] %s' % summary) - else: - DUPLICATE_EVENT_CACHE[dup_string] = 1 - print(summary) + print('Received message {0}:\n{1}'.format( + message.message_id, summarize(message))) subscription.acknowledge([ack_id]) + # [END poll_notifications] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__) + parser.add_argument( + '--project', + help='The project of the subscription, if not in your default project') + parser.add_argument('subscription', + help='The ID of the Pub/Sub subscription') + args = parser.parse_args() + poll_notifications(args.project, args.subscription) diff --git a/storage/cloud-client/notification_polling_test.py b/storage/cloud-client/notification_polling_test.py index ffc4aa5fd156..e21e5b60c137 100644 --- a/storage/cloud-client/notification_polling_test.py +++ b/storage/cloud-client/notification_polling_test.py @@ -15,8 +15,7 @@ from google.cloud.pubsub.message import Message -from notification_polling import GetDedupString -from notification_polling import Summarize +from notification_polling import summarize MESSAGE_ID = 12345 @@ -38,30 +37,11 @@ def test_parse_json_message(): ' "metageneration": 1' '}') message = Message(data, MESSAGE_ID, attributes=attributes) - assert Summarize(message) == ('Object created - mybucket/myobject\n' - '\tGeneration: 1234567\n' - '\tContent type: text/html\n' - '\tSize: 12345\n' - '\tMetageneration: 1\n') - assert GetDedupString(message) == ( - 'OBJECT_FINALIZE|projects/_/buckets/mybucket/' - 'objects/myobject#1234567|1') - - -def test_parse_no_payload_message(): - attributes = { - 'eventType': 'OBJECT_FINALIZE', - 'bucketId': 'mybucket', - 'objectId': 'myobject', - 'objectGeneration': 1234567, - 'resource': 'projects/_/buckets/mybucket/objects/myobject#1234567', - 'notificationConfig': ('projects/_/buckets/mybucket/' - 'notificationConfigs/5'), - 'payloadFormat': 'NONE'} - data = None - message = Message(data, MESSAGE_ID, attributes=attributes) - assert Summarize(message) == ('Object created - mybucket/myobject\n' - '\tGeneration: 1234567\n') - assert GetDedupString(message) == ( - 'OBJECT_FINALIZE|projects/_/buckets/mybucket/' - 'objects/myobject#1234567|unknown') + assert summarize(message) == ( + '\tEvent type: OBJECT_FINALIZE\n' + '\tBucket ID: mybucket\n' + '\tObject ID: myobject\n' + '\tGeneration: 1234567\n' + '\tContent type: text/html\n' + '\tSize: 12345\n' + '\tMetageneration: 1\n') diff --git a/storage/cloud-client/requirements.txt b/storage/cloud-client/requirements.txt index a93b508e2007..7b8c63145d79 100644 --- a/storage/cloud-client/requirements.txt +++ b/storage/cloud-client/requirements.txt @@ -1,3 +1,2 @@ google-cloud-storage==0.22.0 google-cloud-pubsub==0.22.0 -cachetools>=2.0.0 From c9b9bfae4e2aefa58ef5a283d62f5a41d1030716 Mon Sep 17 00:00:00 2001 From: Brandon Yarbrough Date: Fri, 24 Mar 2017 11:09:16 -0700 Subject: [PATCH 06/11] regenerate README --- storage/cloud-client/README.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/storage/cloud-client/README.rst b/storage/cloud-client/README.rst index 0089346172b5..218893be5f74 100644 --- a/storage/cloud-client/README.rst +++ b/storage/cloud-client/README.rst @@ -251,16 +251,16 @@ To run this sample: subscription configured for pull rather than push notifications. If you do not already have notifications configured, you may consult the docs at https://cloud.google.com/storage/docs/reporting-changes or follow the steps - below: 1. [Activate the Google Cloud Pub/Sub API], if you have not already - done so. 2. Create a Google Cloud Storage bucket: ``` $ gsutil mb - gs://testbucket ``` 3. Create a Cloud Pub/Sub topic and publish bucket - notifications there: ``` $ gsutil notification create -f json -t testtopic - gs://testbucket ``` 4. Create a subscription for your new topic: ``` $ gcloud - beta pubsub subscriptions create testsubscription --topic=testtopic ``` 5. Run - this program: ``` $ python notification_polling testsubscription` ``` 6. While - the program is running, upload and delete some files in the testbucket bucket - (you could use the console or gsutil) and watch as changes scroll by in the - app. + below: 1. Activate the Google Cloud Pub/Sub API, if you have not already done + so. https://console.cloud.google.com/flows/enableapi?apiid=pubsub 2. Create a + Google Cloud Storage bucket: $ gsutil mb gs://testbucket 3. Create a Cloud + Pub/Sub topic and publish bucket notifications there: $ gsutil notification + create -f json -t testtopic gs://testbucket 4. Create a subscription for your + new topic: $ gcloud beta pubsub subscriptions create testsubscription + --topic=testtopic 5. Run this program: $ python notification_polling + testsubscription 6. While the program is running, upload and delete some files + in the testbucket bucket (you could use the console or gsutil) and watch as + changes scroll by in the app. positional arguments: subscription The ID of the Pub/Sub subscription From e345075e0261dbcf601aa8f53174ae0c26c8a028 Mon Sep 17 00:00:00 2001 From: Brandon Yarbrough Date: Mon, 27 Mar 2017 11:23:51 -0700 Subject: [PATCH 07/11] Remove explicit project parameter. --- storage/cloud-client/notification_polling.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/storage/cloud-client/notification_polling.py b/storage/cloud-client/notification_polling.py index 9e90f50cea7f..481a98e6697d 100644 --- a/storage/cloud-client/notification_polling.py +++ b/storage/cloud-client/notification_polling.py @@ -87,17 +87,18 @@ def summarize(message): # [END parse_message] -def poll_notifications(project, subscription_id): +def poll_notifications(subscription_id): """Polls a Cloud Pub/Sub subscription for new GCS events for display.""" # [BEGIN poll_notifications] - client = pubsub.Client(project=args.project) + client = pubsub.Client() subscription = pubsub.subscription.Subscription( subscription_id, client=client) + if not subscription.exists(): sys.stderr.write('Cannot find subscription {0}\n'.format(sys.argv[1])) - sys.exit(1) + return - print('Polling for messages.') + print('Polling for messages. Press ctrl+c to exit.') while True: pulled = subscription.pull(max_messages=100) for ack_id, message in pulled: @@ -110,10 +111,7 @@ def poll_notifications(project, subscription_id): if __name__ == '__main__': parser = argparse.ArgumentParser( description=__doc__) - parser.add_argument( - '--project', - help='The project of the subscription, if not in your default project') parser.add_argument('subscription', help='The ID of the Pub/Sub subscription') args = parser.parse_args() - poll_notifications(args.project, args.subscription) + poll_notifications(args.subscription) From 0f42792a5c3dbff41709e90917790ea73bca94a6 Mon Sep 17 00:00:00 2001 From: Brandon Yarbrough Date: Fri, 22 Sep 2017 12:26:39 -0700 Subject: [PATCH 08/11] Fix notification TypeError on start. --- storage/cloud-client/README.rst | 49 +++++++++++++------- storage/cloud-client/notification_polling.py | 15 ++++-- 2 files changed, 44 insertions(+), 20 deletions(-) diff --git a/storage/cloud-client/README.rst b/storage/cloud-client/README.rst index f88c51766dea..fcb2b563f744 100644 --- a/storage/cloud-client/README.rst +++ b/storage/cloud-client/README.rst @@ -225,26 +225,43 @@ To run this sample: $ python notification_polling.py - usage: notification_polling.py [-h] subscription + usage: notification_polling.py [-h] project subscription - This application demonstrates how to poll for GCS notifications from a Cloud - Pub/Sub subscription, parse the incoming message, and acknowledge the - successful processing of the message. This application will work with any - subscription configured for pull rather than push notifications. If you do not - already have notifications configured, you may consult the docs at + This application demonstrates how to poll for GCS notifications from a + Cloud Pub/Sub subscription, parse the incoming message, and acknowledge the + successful processing of the message. + + This application will work with any subscription configured for pull rather + than push notifications. If you do not already have notifications configured, + you may consult the docs at https://cloud.google.com/storage/docs/reporting-changes or follow the steps - below: 1. Activate the Google Cloud Pub/Sub API, if you have not already done - so. https://console.cloud.google.com/flows/enableapi?apiid=pubsub 2. Create a - Google Cloud Storage bucket: $ gsutil mb gs://testbucket 3. Create a Cloud - Pub/Sub topic and publish bucket notifications there: $ gsutil notification - create -f json -t testtopic gs://testbucket 4. Create a subscription for your - new topic: $ gcloud beta pubsub subscriptions create testsubscription - --topic=testtopic 5. Run this program: $ python notification_polling - testsubscription 6. While the program is running, upload and delete some files - in the testbucket bucket (you could use the console or gsutil) and watch as - changes scroll by in the app. + below: + + 1. First, follow the common setup steps for these snippets, specically + configuring auth and installing dependencies. See the README's "Setup" + section. + + 2. Activate the Google Cloud Pub/Sub API, if you have not already done so. + https://console.cloud.google.com/flows/enableapi?apiid=pubsub + + 2. Create a Google Cloud Storage bucket: + $ gsutil mb gs://testbucket + + 3. Create a Cloud Pub/Sub topic and publish bucket notifications there: + $ gsutil notification create -f json -t testtopic gs://testbucket + + 4. Create a subscription for your new topic: + $ gcloud beta pubsub subscriptions create testsubscription --topic=testtopic + + 5. Run this program: + $ python notification_polling my-project-id testsubscription + + 6. While the program is running, upload and delete some files in the testbucket + bucket (you could use the console or gsutil) and watch as changes scroll by + in the app. positional arguments: + project The ID of the project that owns the subscription subscription The ID of the Pub/Sub subscription optional arguments: diff --git a/storage/cloud-client/notification_polling.py b/storage/cloud-client/notification_polling.py index 0580eaea57de..8450030dfc73 100644 --- a/storage/cloud-client/notification_polling.py +++ b/storage/cloud-client/notification_polling.py @@ -24,7 +24,11 @@ https://cloud.google.com/storage/docs/reporting-changes or follow the steps below: -1. Activate the Google Cloud Pub/Sub API, if you have not already done so. +1. First, follow the common setup steps for these snippets, specically + configuring auth and installing dependencies. See the README's "Setup" + section. + +2. Activate the Google Cloud Pub/Sub API, if you have not already done so. https://console.cloud.google.com/flows/enableapi?apiid=pubsub 2. Create a Google Cloud Storage bucket: @@ -37,7 +41,7 @@ $ gcloud beta pubsub subscriptions create testsubscription --topic=testtopic 5. Run this program: - $ python notification_polling testsubscription + $ python notification_polling my-project-id testsubscription 6. While the program is running, upload and delete some files in the testbucket bucket (you could use the console or gsutil) and watch as changes scroll by @@ -110,8 +114,11 @@ def callback(message): if __name__ == '__main__': parser = argparse.ArgumentParser( - description=__doc__) + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('project', + help='The ID of the project that owns the subscription') parser.add_argument('subscription', help='The ID of the Pub/Sub subscription') args = parser.parse_args() - poll_notifications(args.subscription) + poll_notifications(args.project, args.subscription) From 78b3900b3d4a217a74725c17a5cb496ab91d3294 Mon Sep 17 00:00:00 2001 From: Brandon Yarbrough Date: Fri, 22 Sep 2017 12:31:54 -0700 Subject: [PATCH 09/11] Fix linter error. --- storage/cloud-client/notification_polling.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/storage/cloud-client/notification_polling.py b/storage/cloud-client/notification_polling.py index 8450030dfc73..c2cc3ee2c0d2 100644 --- a/storage/cloud-client/notification_polling.py +++ b/storage/cloud-client/notification_polling.py @@ -116,8 +116,9 @@ def callback(message): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('project', - help='The ID of the project that owns the subscription') + parser.add_argument( + 'project', + help='The ID of the project that owns the subscription') parser.add_argument('subscription', help='The ID of the Pub/Sub subscription') args = parser.parse_args() From fd722263cae4ae3e742bb049f3bc7d4270279733 Mon Sep 17 00:00:00 2001 From: Brandon Yarbrough Date: Fri, 22 Sep 2017 13:40:09 -0700 Subject: [PATCH 10/11] Fix ordered list ordinals. --- storage/cloud-client/notification_polling.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/storage/cloud-client/notification_polling.py b/storage/cloud-client/notification_polling.py index c2cc3ee2c0d2..73b921de4c02 100644 --- a/storage/cloud-client/notification_polling.py +++ b/storage/cloud-client/notification_polling.py @@ -31,19 +31,19 @@ 2. Activate the Google Cloud Pub/Sub API, if you have not already done so. https://console.cloud.google.com/flows/enableapi?apiid=pubsub -2. Create a Google Cloud Storage bucket: +3. Create a Google Cloud Storage bucket: $ gsutil mb gs://testbucket -3. Create a Cloud Pub/Sub topic and publish bucket notifications there: +4. Create a Cloud Pub/Sub topic and publish bucket notifications there: $ gsutil notification create -f json -t testtopic gs://testbucket -4. Create a subscription for your new topic: +5. Create a subscription for your new topic: $ gcloud beta pubsub subscriptions create testsubscription --topic=testtopic -5. Run this program: +6. Run this program: $ python notification_polling my-project-id testsubscription -6. While the program is running, upload and delete some files in the testbucket +7. While the program is running, upload and delete some files in the testbucket bucket (you could use the console or gsutil) and watch as changes scroll by in the app. """ From 3a7d2c6652277980be93cf62f079268f2c4fd4be Mon Sep 17 00:00:00 2001 From: Brandon Yarbrough Date: Fri, 22 Sep 2017 13:42:40 -0700 Subject: [PATCH 11/11] Rerun nox readmegen. --- storage/cloud-client/README.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/storage/cloud-client/README.rst b/storage/cloud-client/README.rst index fcb2b563f744..4d9e0a152275 100644 --- a/storage/cloud-client/README.rst +++ b/storage/cloud-client/README.rst @@ -244,19 +244,19 @@ To run this sample: 2. Activate the Google Cloud Pub/Sub API, if you have not already done so. https://console.cloud.google.com/flows/enableapi?apiid=pubsub - 2. Create a Google Cloud Storage bucket: + 3. Create a Google Cloud Storage bucket: $ gsutil mb gs://testbucket - 3. Create a Cloud Pub/Sub topic and publish bucket notifications there: + 4. Create a Cloud Pub/Sub topic and publish bucket notifications there: $ gsutil notification create -f json -t testtopic gs://testbucket - 4. Create a subscription for your new topic: + 5. Create a subscription for your new topic: $ gcloud beta pubsub subscriptions create testsubscription --topic=testtopic - 5. Run this program: + 6. Run this program: $ python notification_polling my-project-id testsubscription - 6. While the program is running, upload and delete some files in the testbucket + 7. While the program is running, upload and delete some files in the testbucket bucket (you could use the console or gsutil) and watch as changes scroll by in the app.