From 8e92bf8cbabb55c908c186716b1ef15021aa4e3e Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Fri, 29 Jan 2016 10:44:57 -0800 Subject: [PATCH] Adding datastore samples. --- .../blog_test.py | 2 + .../wiki_test.py | 2 + datastore/__init__.py | 0 datastore/api/README.md | 4 + datastore/api/__init__.py | 0 datastore/api/index.yaml | 29 + datastore/api/snippets.py | 809 ++++++++++++++++++ datastore/api/snippets_test.py | 266 ++++++ datastore/api/tasks.py | 143 ++++ datastore/api/tasks_test.py | 78 ++ requirements-dev.txt | 1 + tox.ini | 2 + 12 files changed, 1336 insertions(+) create mode 100644 datastore/__init__.py create mode 100644 datastore/api/README.md create mode 100644 datastore/api/__init__.py create mode 100644 datastore/api/index.yaml create mode 100644 datastore/api/snippets.py create mode 100644 datastore/api/snippets_test.py create mode 100644 datastore/api/tasks.py create mode 100644 datastore/api/tasks_test.py diff --git a/blog/introduction_to_data_models_in_cloud_datastore/blog_test.py b/blog/introduction_to_data_models_in_cloud_datastore/blog_test.py index 722ef624f2d0..a0dbcfdceaf5 100644 --- a/blog/introduction_to_data_models_in_cloud_datastore/blog_test.py +++ b/blog/introduction_to_data_models_in_cloud_datastore/blog_test.py @@ -11,11 +11,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from nose.plugins.attrib import attr from tests import CloudBaseTest from .blog import main +@attr('slow') class BlogTestCase(CloudBaseTest): """Simple test case that ensures the blog code doesn't throw any errors.""" diff --git a/blog/introduction_to_data_models_in_cloud_datastore/wiki_test.py b/blog/introduction_to_data_models_in_cloud_datastore/wiki_test.py index aa3aea09676a..53f2e059a47a 100644 --- a/blog/introduction_to_data_models_in_cloud_datastore/wiki_test.py +++ b/blog/introduction_to_data_models_in_cloud_datastore/wiki_test.py @@ -11,11 +11,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from nose.plugins.attrib import attr from tests import CloudBaseTest from .wiki import main +@attr('slow') class WikiTestCase(CloudBaseTest): """Simple test case that ensures the wiki code doesn't throw any errors.""" diff --git a/datastore/__init__.py b/datastore/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/datastore/api/README.md b/datastore/api/README.md new file mode 100644 index 000000000000..a53482eda9d6 --- /dev/null +++ b/datastore/api/README.md @@ -0,0 +1,4 @@ +# Cloud Datastore API Samples + + + diff --git a/datastore/api/__init__.py b/datastore/api/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/datastore/api/index.yaml b/datastore/api/index.yaml new file mode 100644 index 000000000000..8077e5fa015f --- /dev/null +++ b/datastore/api/index.yaml @@ -0,0 +1,29 @@ +indexes: +- kind: Task + properties: + - name: done + - name: priority + direction: desc +- kind: Task + properties: + - name: priority + - name: percent_complete +- kind: Task + properties: + - name: priority + direction: desc + - name: created +- kind: Task + properties: + - name: priority + - name: created +- kind: Task + properties: + - name: type + - name: priority +- kind: Task + properties: + - name: priority + - name: done + - name: created + direction: desc diff --git a/datastore/api/snippets.py b/datastore/api/snippets.py new file mode 100644 index 000000000000..d58b2aa0de74 --- /dev/null +++ b/datastore/api/snippets.py @@ -0,0 +1,809 @@ +# Copyright 2016, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +from collections import defaultdict +import datetime +from pprint import pprint + +import gcloud +from gcloud import datastore + + +def incomplete_key(client): + # [START incomplete_key] + key = client.key('Task') + # [END incomplete_key] + + return key + + +def named_key(client): + # [START named_key] + key = client.key('Task', 'sample_task') + # [END named_key] + + return key + + +def key_with_parent(client): + # [START key_with_parent] + key = client.key('TaskList', 'default', 'Task', 'sample_task') + # Alternatively + parent_key = client.key('TaskList', 'default') + key = client.key('Task', 'sample_task', parent=parent_key) + # [END key_with_parent] + + return key + + +def key_with_multilevel_parent(client): + # [START key_with_multilevel_parent] + key = client.key( + 'User', 'alice', + 'TaskList', 'default', + 'Task', 'sample_task') + # [END key_with_multilevel_parent] + + return key + + +def basic_entity(client): + # [START basic_entity] + task = datastore.Entity(client.key('Task')) + task.update({ + 'type': 'Personal', + 'done': False, + 'priority': 4, + 'description': 'Learn Cloud Datastore' + }) + # [END basic_entity] + + return task + + +def entity_with_parent(client): + # [START entity_with_parent] + key_with_parent = client.key( + 'TaskList', 'default', 'Task', 'sample_task') + + task = datastore.Entity(key=key_with_parent) + + task.update({ + 'type': 'Personal', + 'done': False, + 'priority': 4, + 'description': 'Learn Cloud Datastore' + }) + # [END entity_with_parent] + + return task + + +def properties(client): + key = client.key('Task') + # [START properties] + task = datastore.Entity( + key, + exclude_from_indexes=['description']) + task.update({ + 'type': 'Personal', + 'description': 'Learn Cloud Datastore', + 'created': datetime.datetime.utcnow(), + 'done': False, + 'priority': 4, + 'percent_complete': 10.5, + }) + # [END properties] + + return task + + +def array_value(client): + key = client.key('Task') + # [START array_value] + task = datastore.Entity(key) + task.update({ + 'tags': [ + 'fun', + 'programming' + ], + 'collaborators': [ + 'alice', + 'bob' + ] + }) + # [END array_value] + + return task + + +def upsert(client): + # [START upsert] + complete_key = client.key('Task', 'sample_task') + + task = datastore.Entity(key=complete_key) + + task.update({ + 'type': 'Personal', + 'done': False, + 'priority': 4, + 'description': 'Learn Cloud Datastore' + }) + + client.put(task) + # [END upsert] + + return task + + +def insert(client): + # [START insert] + with client.transaction(): + incomplete_key = client.key('Task') + + task = datastore.Entity(key=incomplete_key) + + task.update({ + 'type': 'Personal', + 'done': False, + 'priority': 4, + 'description': 'Learn Cloud Datastore' + }) + + client.put(task) + # [END insert] + + return task + + +def update(client): + # Create the entity we're going to update. + upsert(client) + + # [START update] + with client.transaction(): + key = client.key('Task', 'sample_task') + task = client.get(key) + + task['done'] = True + + client.put(task) + # [END update] + + return task + + +def lookup(client): + # Create the entity that we're going to look up. + upsert(client) + + # [START lookup] + key = client.key('Task', 'sample_task') + task = client.get(key) + # [END lookup] + + return task + + +def delete(client): + # Create the entity we're going to delete. + upsert(client) + + # [START delete] + key = client.key('Task', 'sample_task') + client.delete(key) + # [END delete] + + return key + + +def batch_upsert(client): + task1 = datastore.Entity(client.key('Task', 1)) + + task1.update({ + 'type': 'Personal', + 'done': False, + 'priority': 4, + 'description': 'Learn Cloud Datastore' + }) + + task2 = datastore.Entity(client.key('Task', 2)) + + task2.update({ + 'type': 'Work', + 'done': False, + 'priority': 8, + 'description': 'Integrate Cloud Datastore' + }) + + # [START batch_upsert] + client.put_multi([task1, task2]) + # [END batch_upsert] + + return task1, task2 + + +def batch_lookup(client): + # Create the entities we will lookup. + batch_upsert(client) + + keys = [ + client.key('Task', 1), + client.key('Task', 2) + ] + + # [START batch_lookup] + tasks = client.get_multi(keys) + # [END batch_lookup] + + return tasks + + +def batch_delete(client): + # Create the entities we will delete. + batch_upsert(client) + + keys = [ + client.key('Task', 1), + client.key('Task', 2) + ] + + # [START batch_delete] + client.delete_multi(keys) + # [END batch_delete] + + return keys + + +def unindexed_property_query(client): + # Create the entity that we're going to query. + upsert(client) + + # [START unindexed_property_query] + query = client.query(kind='Task') + query.add_filter('description', '=', 'Learn Cloud Datastore') + # [END unindexed_property_query] + + return list(query.fetch()) + + +def basic_query(client): + # Create the entity that we're going to query. + upsert(client) + + # [START basic_query] + query = client.query(kind='Task') + query.add_filter('done', '=', False) + query.add_filter('priority', '>=', 4) + query.order = ['-priority'] + # [END basic_query] + + return list(query.fetch()) + + +def projection_query(client): + # Create the entity that we're going to query. + task = datastore.Entity(client.key('Task')) + task.update({ + 'type': 'Personal', + 'done': False, + 'priority': 4, + 'description': 'Learn Cloud Datastore', + 'percent_complete': 0.5 + }) + client.put(task) + + # [START projection_query] + query = client.query(kind='Task') + query.projection = ['priority', 'percent_complete'] + # [END projection_query] + + # [START run_query_projection] + priorities = [] + percent_completes = [] + + for task in query.fetch(): + priorities.append(task['priority']) + percent_completes.append(task['priority']) + # [END run_query_projection] + + return priorities, percent_completes + + +def ancestor_query(client): + task = datastore.Entity( + client.key('TaskList', 'default', 'Task')) + task.update({ + 'type': 'Personal', + 'description': 'Learn Cloud Datastore', + }) + client.put(task) + + # [START ancestor_query] + ancestor = client.key('TaskList', 'default') + query = client.query(kind='Task', ancestor=ancestor) + # [END ancestor_query] + + return list(query.fetch()) + + +def run_query(client): + # [START run_query] + query = client.query() + results = list(query.fetch()) + # [END run_query] + + return results + + +def limit(client): + # [START limit] + query = client.query() + tasks = list(query.fetch(limit=5)) + # [END limit] + + return tasks + + +def cursor_paging(client): + # [START cursor_paging] + + def get_one_page_of_tasks(cursor=None): + query = client.query(kind='Task') + query_iter = query.fetch(start_cursor=cursor, limit=5) + tasks, _, cursor = query_iter.next_page() + + return tasks, cursor + # [END cursor_paging] + + page_one, cursor_one = get_one_page_of_tasks() + page_two, cursor_two = get_one_page_of_tasks(cursor=cursor_one) + return page_one, cursor_one, page_two, cursor_two + + +def property_filter(client): + # Create the entity that we're going to query. + upsert(client) + + # [START property_filter] + query = client.query(kind='Task') + query.add_filter('done', '=', False) + # [END property_filter] + + return list(query.fetch()) + + +def composite_filter(client): + # Create the entity that we're going to query. + upsert(client) + + # [START composite_filter] + query = client.query(kind='Task') + query.add_filter('done', '=', False) + query.add_filter('priority', '=', 4) + # [END composite_filter] + + return list(query.fetch()) + + +def key_filter(client): + # Create the entity that we're going to query. + upsert(client) + + # [START key_filter] + query = client.query(kind='Task') + first_key = client.key('Task', 'first_task') + query.add_filter('__key__', '>', first_key) + # [END key_filter] + + return list(query.fetch()) + + +def ascending_sort(client): + # Create the entity that we're going to query. + task = upsert(client) + task['created'] = datetime.datetime.utcnow() + client.put(task) + + # [START ascending_sort] + query = client.query(kind='Task') + query.order = ['created'] + # [END ascending_sort] + + return list(query.fetch()) + + +def descending_sort(client): + # Create the entity that we're going to query. + task = upsert(client) + task['created'] = datetime.datetime.utcnow() + client.put(task) + + # [START descending_sort] + query = client.query(kind='Task') + query.order = ['-created'] + # [END descending_sort] + + return list(query.fetch()) + + +def multi_sort(client): + # Create the entity that we're going to query. + task = upsert(client) + task['created'] = datetime.datetime.utcnow() + client.put(task) + + # [START multi_sort] + query = client.query(kind='Task') + query.order = [ + '-priority', + 'created' + ] + # [END multi_sort] + + return list(query.fetch()) + + +def keys_only_query(client): + # Create the entity that we're going to query. + upsert(client) + + # [START keys_only_query] + query = client.query() + query.keys_only() + # [END keys_only_query] + + # [START run_keys_only_query] + keys = list([entity.key for entity in query.fetch(limit=10)]) + # [END run_keys_only_query] + + return keys + + +def distinct_query(client): + # Create the entity that we're going to query. + upsert(client) + + # [START distinct_query] + query = client.query(kind='Task') + query.group_by = ['type', 'priority'] + query.order = ['type', 'priority'] + query.projection = ['type', 'priority'] + # [END distinct_query] + + return list(query.fetch()) + + +def distinct_on_query(client): + # Create the entity that we're going to query. + upsert(client) + + # [START distinct_on_query] + query = client.query(kind='Task') + query.group_by = ['type'] + query.order = ['type', 'priority'] + # [END distinct_on_query] + + return list(query.fetch()) + + +def kindless_query(client): + # Create the entity that we're going to query. + upsert(client) + + last_seen_key = client.key('Task', 'a') + + # [START kindless_query] + query = client.query() + query.add_filter('__key__', '>', last_seen_key) + # [END kindless_query] + + return list(query.fetch()) + + +def inequality_range(client): + # [START inequality_range] + start_date = datetime.datetime(1990, 1, 1) + end_date = datetime.datetime(2000, 1, 1) + query = client.query(kind='Task') + query.add_filter( + 'created', '>', start_date) + query.add_filter( + 'created', '<', end_date) + # [END inequality_range] + + return list(query.fetch()) + + +def inequality_invalid(client): + try: + # [START inequality_invalid] + start_date = datetime.datetime(1990, 1, 1) + query = client.query(kind='Task') + query.add_filter( + 'created', '>', start_date) + query.add_filter( + 'priority', '>', 3) + # [END inequality_invalid] + + return list(query.fetch()) + + except gcloud.exceptions.BadRequest: + pass + + +def equal_and_inequality_range(client): + # [START equal_and_inequality_range] + start_date = datetime.datetime(1990, 1, 1) + end_date = datetime.datetime(2000, 12, 31, 23, 59, 59) + query = client.query(kind='Task') + query.add_filter('priority', '=', 4) + query.add_filter('done', '=', False) + query.add_filter( + 'created', '>', start_date) + query.add_filter( + 'created', '<', end_date) + # [END equal_and_inequality_range] + + return list(query.fetch()) + + +def inequality_sort(client): + # [START inequality_sort] + query = client.query(kind='Task') + query.add_filter('priority', '>', 3) + query.order = ['priority', 'created'] + # [END inequality_sort] + + return list(query.fetch()) + + +def inequality_sort_invalid_not_same(client): + try: + # [START inequality_sort_invalid_not_same] + query = client.query(kind='Task') + query.add_filter('priority', '>', 3) + query.order = ['created'] + # [END inequality_sort_invalid_not_same] + + return list(query.fetch()) + + except gcloud.exceptions.BadRequest: + pass + + +def inequality_sort_invalid_not_first(client): + try: + # [START inequality_sort_invalid_not_first] + query = client.query(kind='Task') + query.add_filter('priority', '>', 3) + query.order = ['created', 'priority'] + # [END inequality_sort_invalid_not_first] + + return list(query.fetch()) + + except gcloud.exceptions.BadRequest: + pass + + +def array_value_inequality_range(client): + # [START array_value_inequality_range] + query = client.query(kind='Task') + query.add_filter('tag', '>', 'learn') + query.add_filter('tag', '<', 'math') + # [END array_value_inequality_range] + + return list(query.fetch()) + + +def array_value_equality(client): + # [START array_value_equality] + query = client.query(kind='Task') + query.add_filter('tag', '=', 'fun') + query.add_filter('tag', '=', 'programming') + # [END array_value_equality] + + return list(query.fetch()) + + +def exploding_properties(client): + # [START exploding_properties] + task = datastore.Entity(client.key('Task')) + task.update({ + 'tags': [ + 'fun', + 'programming', + 'learn' + ], + 'collaborators': [ + 'alice', + 'bob', + 'charlie' + ], + 'created': datetime.datetime.utcnow() + }) + # [END exploding_properties] + + return task + + +def transactional_update(client): + # Create the entities we're going to manipulate + account1 = datastore.Entity(client.key('Account')) + account1['balance'] = 100 + account2 = datastore.Entity(client.key('Account')) + account2['balance'] = 100 + client.put_multi([account1, account2]) + + # [START transactional_update] + def transfer_funds(client, from_key, to_key, amount): + with client.transaction(): + from_account, to_account = client.get_multi([from_key, to_key]) + + from_account['balance'] -= amount + to_account['balance'] += amount + + client.put_multi([from_account, to_account]) + # [END transactional_update] + + # [START transactional_retry] + for _ in range(5): + try: + transfer_funds(client, account1.key, account2.key, 50) + except gcloud.exceptions.Conflict: + continue + # [END transaction_retry] + + return account1.key, account2.key + + +def transactional_get_or_create(client): + # [START transactional_get_or_create] + with client.transaction(): + key = client.key('Task', datetime.datetime.utcnow().isoformat()) + + task = client.get(key) + + if not task: + task = datastore.Entity(key) + task.update({ + 'description': 'Example task' + }) + client.put(task) + + return task + # [END transactional_get_or_create] + + +def transactional_single_entity_group_read_only(client): + client.put_multi([ + datastore.Entity(key=client.key('TaskList', 'default')), + datastore.Entity(key=client.key('TaskList', 'default', 'Task', 1)) + ]) + + # [START transactional_single_entity_group_read_only] + with client.transaction(): + task_list_key = client.key('TaskList', 'default') + + task_list = client.get(task_list_key) + + query = client.query(kind='Task', ancestor=task_list_key) + tasks_in_list = list(query.fetch()) + + return task_list, tasks_in_list + # [END transactional_single_entity_group_read_only] + + +def namespace_run_query(client): + # Create an entity in another namespace. + task = datastore.Entity( + client.key('Task', 'sample-task', namespace='google')) + client.put(task) + + # [START namespace_run_query] + # All namespaces + query = client.query(kind='__namespace__') + query.keys_only() + + all_namespaces = [entity.key.id_or_name for entity in query.fetch()] + + # Filtered namespaces + start_namespace = client.key('__namespace__', 'g') + end_namespace = client.key('__namespace__', 'h') + query = client.query(kind='__namespace__') + query.add_filter( + '__key__', '>=', start_namespace) + query.add_filter( + '__key__', '<', end_namespace) + + filtered_namespaces = [entity.key.id_or_name for entity in query.fetch()] + # [END namespace_run_query] + + return all_namespaces, filtered_namespaces + + +def kind_run_query(client): + # Create the entity that we're going to query. + upsert(client) + + # [START kind_run_query] + query = client.query(kind='__kind__') + query.keys_only() + + kinds = [entity.key.id_or_name for entity in query.fetch()] + # [END kind_run_query] + + return kinds + + +def property_run_query(client): + # Create the entity that we're going to query. + upsert(client) + + # [START property_run_query] + query = client.query(kind='__property__') + query.keys_only() + + properties_by_kind = defaultdict(list) + + for entity in query.fetch(): + kind = entity.key.parent.name + property_ = entity.key.name + + properties_by_kind[kind].append(property_) + # [END property_run_query] + + return properties_by_kind + + +def property_by_kind_run_query(client): + # Create the entity that we're going to query. + upsert(client) + + # [START property_by_kind_run_query] + ancestor = client.key('__kind__', 'Task') + query = client.query(kind='__property__', ancestor=ancestor) + + representations_by_property = {} + + for entity in query.fetch(): + property_name = entity.key.name + property_types = entity['property_representation'] + + representations_by_property[property_name] = property_types + # [END property_by_kind_run_query] + + return representations_by_property + + +def main(project_id): + client = datastore.Client(project_id) + + for name, function in globals().iteritems(): + if name in ('main', 'defaultdict') or not callable(function): + continue + + print(name) + pprint(function(client)) + print('\n-----------------\n') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Demonstrates datastore API operations.') + parser.add_argument('project_id', help='Your cloud project ID.') + + args = parser.parse_args() + + main(args.project_id) diff --git a/datastore/api/snippets_test.py b/datastore/api/snippets_test.py new file mode 100644 index 000000000000..17eed9613510 --- /dev/null +++ b/datastore/api/snippets_test.py @@ -0,0 +1,266 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from flaky import flaky +import gcloud +from gcloud import datastore +from nose.plugins.attrib import attr +from tests import CloudBaseTest + +from . import snippets + + +def flaky_filter(e, *args): + return isinstance(e, gcloud.exceptions.GCloudError) + + +@attr('slow') +@flaky(rerun_filter=flaky_filter) +class DatastoreSnippetsTest(CloudBaseTest): + + def setUp(self): + super(DatastoreSnippetsTest, self).setUp() + self.client = datastore.Client(self.project_id) + self.to_delete_entities = [] + self.to_delete_keys = [] + + def tearDown(self): + super(DatastoreSnippetsTest, self).tearDown() + with self.client.batch(): + self.client.delete_multi( + [x.key for x in self.to_delete_entities] + self.to_delete_keys) + + # These tests mostly just test the absence of exceptions. + + def test_incomplete_key(self): + self.assertTrue( + snippets.incomplete_key(self.client)) + + def test_named_key(self): + self.assertTrue( + snippets.named_key(self.client)) + + def test_key_with_parent(self): + self.assertTrue( + snippets.key_with_parent(self.client)) + + def test_key_with_multilevel_parent(self): + self.assertTrue( + snippets.key_with_multilevel_parent(self.client)) + + def test_basic_entity(self): + self.assertTrue( + snippets.basic_entity(self.client)) + + def test_entity_with_parent(self): + self.assertTrue( + snippets.entity_with_parent(self.client)) + + def test_properties(self): + self.assertTrue( + snippets.properties(self.client)) + + def test_array_value(self): + self.assertTrue( + snippets.array_value(self.client)) + + def test_upsert(self): + task = snippets.upsert(self.client) + self.to_delete_entities.append(task) + self.assertTrue(task) + + def test_insert(self): + task = snippets.insert(self.client) + self.to_delete_entities.append(task) + self.assertTrue(task) + + def test_update(self): + task = snippets.insert(self.client) + self.to_delete_entities.append(task) + self.assertTrue(task) + + def test_lookup(self): + task = snippets.lookup(self.client) + self.to_delete_entities.append(task) + self.assertTrue(task) + + def test_delete(self): + snippets.delete(self.client) + + def test_batch_upsert(self): + tasks = snippets.batch_upsert(self.client) + self.to_delete_entities.extend(tasks) + self.assertTrue(tasks) + + def test_batch_lookup(self): + tasks = snippets.batch_lookup(self.client) + self.to_delete_entities.extend(tasks) + self.assertTrue(tasks) + + def test_batch_delete(self): + snippets.batch_delete(self.client) + + def test_unindexed_property_query(self): + tasks = snippets.unindexed_property_query(self.client) + self.to_delete_entities.extend(tasks) + self.assertTrue(tasks) + + def test_basic_query(self): + tasks = snippets.basic_query(self.client) + self.to_delete_entities.extend(tasks) + self.assertTrue(tasks) + + def test_projection_query(self): + priorities, percents = snippets.projection_query(self.client) + self.to_delete_entities.extend(self.client.query(kind='Task').fetch()) + self.assertTrue(priorities) + self.assertTrue(percents) + + def test_ancestor_query(self): + tasks = snippets.ancestor_query(self.client) + self.to_delete_entities.extend(tasks) + self.assertTrue(tasks) + + def test_run_query(self): + snippets.run_query(self.client) + + def test_cursor_paging(self): + for n in range(6): + self.to_delete_entities.append( + snippets.insert(self.client)) + + page_one, cursor_one, page_two, cursor_two = snippets.cursor_paging( + self.client) + + self.assertTrue(len(page_one) == 5) + self.assertTrue(len(page_two) == 1) + self.assertTrue(cursor_one) + self.assertTrue(cursor_two) + + def test_property_filter(self): + tasks = snippets.property_filter(self.client) + self.to_delete_entities.extend(tasks) + self.assertTrue(tasks) + + def test_composite_filter(self): + tasks = snippets.composite_filter(self.client) + self.to_delete_entities.extend(tasks) + self.assertTrue(tasks) + + def test_key_filter(self): + tasks = snippets.key_filter(self.client) + self.to_delete_entities.extend(tasks) + self.assertTrue(tasks) + + def test_ascending_sort(self): + tasks = snippets.ascending_sort(self.client) + self.to_delete_entities.extend(tasks) + self.assertTrue(tasks) + + def test_descending_sort(self): + tasks = snippets.descending_sort(self.client) + self.to_delete_entities.extend(tasks) + self.assertTrue(tasks) + + def test_multi_sort(self): + tasks = snippets.multi_sort(self.client) + self.to_delete_entities.extend(tasks) + self.assertTrue(tasks) + + def test_keys_only_query(self): + keys = snippets.keys_only_query(self.client) + self.to_delete_keys.extend(keys) + self.assertTrue(keys) + + def test_distinct_query(self): + tasks = snippets.distinct_query(self.client) + self.to_delete_entities.extend(tasks) + self.assertTrue(tasks) + + def test_distinct_on_query(self): + tasks = snippets.distinct_on_query(self.client) + self.to_delete_entities.extend(tasks) + self.assertTrue(tasks) + + def test_kindless_query(self): + tasks = snippets.kindless_query(self.client) + self.assertTrue(tasks) + + def test_inequality_range(self): + snippets.inequality_range(self.client) + + def test_inequality_invalid(self): + snippets.inequality_invalid(self.client) + + def test_equal_and_inequality_range(self): + snippets.equal_and_inequality_range(self.client) + + def test_inequality_sort(self): + snippets.inequality_sort(self.client) + + def test_inequality_sort_invalid_not_same(self): + snippets.inequality_sort_invalid_not_same(self.client) + + def test_inequality_sort_invalid_not_first(self): + snippets.inequality_sort_invalid_not_first(self.client) + + def test_array_value_inequality_range(self): + snippets.array_value_inequality_range(self.client) + + def test_array_value_equality(self): + snippets.array_value_equality(self.client) + + def test_exploding_properties(self): + task = snippets.exploding_properties(self.client) + self.assertTrue(task) + + def test_transactional_update(self): + keys = snippets.transactional_update(self.client) + self.to_delete_keys.extend(keys) + + def test_transactional_get_or_create(self): + task = snippets.transactional_get_or_create(self.client) + self.to_delete_entities.append(task) + self.assertTrue(task) + + def transactional_single_entity_group_read_only(self): + task_list, tasks_in_list = \ + snippets.transactional_single_entity_group_read_only(self.client) + self.to_delete_entities.append(task_list) + self.to_delete_entities.extend(tasks_in_list) + self.assertTrue(task_list) + self.assertTrue(tasks_in_list) + + def test_namespace_run_query(self): + all_namespaces, filtered_namespaces = snippets.namespace_run_query( + self.client) + self.assertTrue(all_namespaces) + self.assertTrue(filtered_namespaces) + self.assertTrue('google' in filtered_namespaces) + + def test_kind_run_query(self): + kinds = snippets.kind_run_query(self.client) + self.to_delete_entities.extend(self.client.query(kind='Task').fetch()) + self.assertTrue(kinds) + self.assertTrue('Task' in kinds) + + def test_property_run_query(self): + kinds = snippets.property_run_query(self.client) + self.to_delete_entities.extend(self.client.query(kind='Task').fetch()) + self.assertTrue(kinds) + self.assertTrue('Task' in kinds) + + def test_property_by_kind_run_query(self): + reprs = snippets.property_by_kind_run_query(self.client) + self.to_delete_entities.extend(self.client.query(kind='Task').fetch()) + self.assertTrue(reprs) diff --git a/datastore/api/tasks.py b/datastore/api/tasks.py new file mode 100644 index 000000000000..164b4bc2c861 --- /dev/null +++ b/datastore/api/tasks.py @@ -0,0 +1,143 @@ +# Copyright 2016, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import datetime + +import gcloud + +# [START build_service] +from gcloud import datastore + + +def create_client(project_id): + return gcloud.datastore.Client(project_id) +# [END build_service] + + +# [START add_entity] +def add_task(client, description): + key = client.key('Task') + + task = datastore.Entity( + key, exclude_from_indexes=['description']) + + task.update({ + 'created': datetime.datetime.utcnow(), + 'description': description, + 'done': False + }) + + client.put(task) + + return task.key +# [END add_entity] + + +# [START update_entity] +def mark_done(client, task_id): + with client.transaction(): + key = client.key('Task', task_id) + task = client.get(key) + + if not task: + raise ValueError( + 'Task {} does not exist.'.format(task_id)) + + task['done'] = True + + client.put(task) +# [END update_entity] + + +# [START retrieve_entities] +def list_tasks(client): + query = client.query(kind='Task') + query.order = ['created'] + + return list(query.fetch()) +# [END retrieve_entities] + + +# [START delete_entity] +def delete_task(client, task_id): + key = client.key('Task', task_id) + client.delete(key) +# [END delete_entity] + + +# [START format_results] +def format_tasks(tasks): + lines = [] + for task in tasks: + if task['done']: + status = 'done' + else: + status = 'created {}'.format(task['created']) + + lines.append('{}: {} ({})'.format( + task.key.id, task['description'], status)) + + return '\n'.join(lines) +# [END format_results] + + +def new_command(client, args): + """Adds a task with description .""" + task_key = add_task(client, args.description) + print('Task {} added.'.format(task_key.id)) + + +def done_command(client, args): + """Marks a task as done.""" + mark_done(client, args.task_id) + print('Task {} marked done.'.format(args.task_id)) + + +def list_command(client, args): + """Lists all tasks by creation time.""" + print(format_tasks(list_tasks(client))) + + +def delete_command(client, args): + """Deletes a task.""" + mark_done(client, args.task_id) + print('Task {} deleted.'.format(args.task_id)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + + parser.add_argument('--project-id', help='Your cloud project ID.') + + new_parser = subparsers.add_parser('new', help=new_command.__doc__) + new_parser.set_defaults(func=new_command) + new_parser.add_argument('description', help='New task description.') + + done_parser = subparsers.add_parser('done', help=done_command.__doc__) + done_parser.set_defaults(func=done_command) + done_parser.add_argument('task_id', help='Task ID.', type=int) + + list_parser = subparsers.add_parser('list', help=list_command.__doc__) + list_parser.set_defaults(func=list_command) + + delete_parser = subparsers.add_parser( + 'delete', help=delete_command.__doc__) + delete_parser.set_defaults(func=delete_command) + delete_parser.add_argument('task_id', help='Task ID.', type=int) + + args = parser.parse_args() + + client = create_client(args.project_id) + args.func(client, args) diff --git a/datastore/api/tasks_test.py b/datastore/api/tasks_test.py new file mode 100644 index 000000000000..ed71e0c8a062 --- /dev/null +++ b/datastore/api/tasks_test.py @@ -0,0 +1,78 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from flaky import flaky +import gcloud +from gcloud import datastore +from nose.plugins.attrib import attr +from tests import CloudBaseTest + +from . import tasks + + +def flaky_filter(e, *args): + return isinstance(e, gcloud.exceptions.GCloudError) + + +@attr('slow') +@flaky(rerun_filter=flaky_filter) +class DatastoreTasksTest(CloudBaseTest): + + def setUp(self): + super(DatastoreTasksTest, self).setUp() + self.client = datastore.Client(self.project_id) + + def tearDown(self): + super(DatastoreTasksTest, self).tearDown() + with self.client.batch(): + self.client.delete_multi( + [x.key for x in self.client.query(kind='Task').fetch()]) + + def test_create_client(self): + tasks.create_client(self.project_id) + + def test_add_task(self): + task_key = tasks.add_task(self.client, 'Test task') + task = self.client.get(task_key) + self.assertTrue(task) + self.assertEqual(task['description'], 'Test task') + + def test_mark_done(self): + task_key = tasks.add_task(self.client, 'Test task') + tasks.mark_done(self.client, task_key.id) + task = self.client.get(task_key) + self.assertTrue(task) + self.assertTrue(task['done']) + + def test_list_tasks(self): + task1_key = tasks.add_task(self.client, 'Test task 1') + task2_key = tasks.add_task(self.client, 'Test task 2') + task_list = tasks.list_tasks(self.client) + self.assertEqual([x.key for x in task_list], [task1_key, task2_key]) + + def test_delete_task(self): + task_key = tasks.add_task(self.client, 'Test task 1') + tasks.delete_task(self.client, task_key.id) + self.assertIsNone(self.client.get(task_key)) + + def test_format_tasks(self): + task1_key = tasks.add_task(self.client, 'Test task 1') + tasks.add_task(self.client, 'Test task 2') + tasks.mark_done(self.client, task1_key.id) + + output = tasks.format_tasks(tasks.list_tasks(self.client)) + + self.assertTrue('Test task 1' in output) + self.assertTrue('Test task 2' in output) + self.assertTrue('done' in output) + self.assertTrue('created' in output) diff --git a/requirements-dev.txt b/requirements-dev.txt index 9f9680b355bb..e4b177acd4c7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -20,3 +20,4 @@ Flask-SQLAlchemy==2.0 PyMySQL==0.6.6 python-memcached==1.57 PyCrypto==2.6.1 +flaky==3.0.3 diff --git a/tox.ini b/tox.ini index e35ad5ffd6ea..0473f848a7e7 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,8 @@ commonargs = --cover-tests --cover-branches --cover-inclusive + --with-flaky + --no-success-flaky-report [testenv:reqcheck] deps =