From 88cb861926f9d214dac55820e477836ecf7b918d Mon Sep 17 00:00:00 2001 From: larkee <31196561+larkee@users.noreply.github.com> Date: Mon, 30 Mar 2020 15:41:06 +1300 Subject: [PATCH] spanner: Add Cloud Spanner Backup samples (#3101) * add backup samples * update required spanner version * fix lint errors * run backup samples tests against a new instance * fix lint * wait for instance creation to complete * Apply suggestions from code review Co-Authored-By: skuruppu * add list_backups test * fix lint * add missing newline character in assert * update samples to be consistent with other languages * lint fix * add pagination sample * reorder tests Co-authored-by: larkee Co-authored-by: skuruppu --- spanner/cloud-client/backup_sample.py | 281 +++++++++++++++++++++ spanner/cloud-client/backup_sample_test.py | 116 +++++++++ 2 files changed, 397 insertions(+) create mode 100644 spanner/cloud-client/backup_sample.py create mode 100644 spanner/cloud-client/backup_sample_test.py diff --git a/spanner/cloud-client/backup_sample.py b/spanner/cloud-client/backup_sample.py new file mode 100644 index 000000000000..c8bcce82b835 --- /dev/null +++ b/spanner/cloud-client/backup_sample.py @@ -0,0 +1,281 @@ +# Copyright 2020 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to create and restore from backups +using Cloud Spanner. + +For more information, see the README.rst under /spanner. +""" + +import argparse +from datetime import ( + datetime, + timedelta +) +import time + +from google.cloud import spanner + + +# [START spanner_create_backup] +def create_backup(instance_id, database_id, backup_id): + """Creates a backup for a database.""" + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + # Create a backup + expire_time = datetime.utcnow() + timedelta(days=14) + backup = instance.backup( + backup_id, database=database, expire_time=expire_time) + operation = backup.create() + + # Wait for backup operation to complete. + operation.result() + + # Verify that the backup is ready. + backup.reload() + assert backup.is_ready() is True + + # Get the name, create time and backup size. + backup.reload() + print("Backup {} of size {} bytes was created at {}".format( + backup.name, backup.size_bytes, backup.create_time)) +# [END spanner_create_backup] + + +# [START spanner_restore_database] +def restore_database(instance_id, new_database_id, backup_id): + """Restores a database from a backup.""" + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + # Create a backup on database_id. + + # Start restoring backup to a new database. + backup = instance.backup(backup_id) + new_database = instance.database(new_database_id) + operation = new_database.restore(backup) + + # Wait for restore operation to complete. + operation.result() + + # Newly created database has restore information. + new_database.reload() + restore_info = new_database.restore_info + print("Database {} restored to {} from backup {}.".format( + restore_info.backup_info.source_database, + new_database_id, + restore_info.backup_info.backup)) +# [END spanner_restore_database] + + +# [START spanner_cancel_backup] +def cancel_backup(instance_id, database_id, backup_id): + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + expire_time = datetime.utcnow() + timedelta(days=30) + + # Create a backup. + backup = instance.backup( + backup_id, database=database, expire_time=expire_time) + operation = backup.create() + + # Cancel backup creation. + operation.cancel() + + # Cancel operations are best effort so either it will complete or + # be cancelled. + while not operation.done(): + time.sleep(300) # 5 mins + + # Deal with resource if the operation succeeded. + if backup.exists(): + print("Backup was created before the cancel completed.") + backup.delete() + print("Backup deleted.") + else: + print("Backup creation was successfully cancelled.") +# [END spanner_cancel_backup] + + +# [START spanner_list_backup_operations] +def list_backup_operations(instance_id, database_id): + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + + # List the CreateBackup operations. + filter_ = ( + "(metadata.database:{}) AND " + "(metadata.@type:type.googleapis.com/" + "google.spanner.admin.database.v1.CreateBackupMetadata)" + ).format(database_id) + operations = instance.list_backup_operations(filter_=filter_) + for op in operations: + metadata = op.metadata + print("Backup {} on database {}: {}% complete.".format( + metadata.name, metadata.database, + metadata.progress.progress_percent)) +# [END spanner_list_backup_operations] + + +# [START spanner_list_database_operations] +def list_database_operations(instance_id): + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + + # List the progress of restore. + filter_ = ( + "(metadata.@type:type.googleapis.com/" + "google.spanner.admin.database.v1.OptimizeRestoredDatabaseMetadata)" + ) + operations = instance.list_database_operations(filter_=filter_) + for op in operations: + print("Database {} restored from backup is {}% optimized.".format( + op.metadata.name, op.metadata.progress.progress_percent)) +# [END spanner_list_database_operations] + + +# [START spanner_list_backups] +def list_backups(instance_id, database_id, backup_id): + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + + # List all backups. + print("All backups:") + for backup in instance.list_backups(): + print(backup.name) + + # List all backups that contain a name. + print("All backups with backup name containing \"{}\":".format(backup_id)) + for backup in instance.list_backups(filter_="name:{}".format(backup_id)): + print(backup.name) + + # List all backups for a database that contains a name. + print("All backups with database name containing \"{}\":".format(database_id)) + for backup in instance.list_backups(filter_="database:{}".format(database_id)): + print(backup.name) + + # List all backups that expire before a timestamp. + expire_time = datetime.utcnow().replace(microsecond=0) + timedelta(days=30) + print("All backups with expire_time before \"{}-{}-{}T{}:{}:{}Z\":".format( + *expire_time.timetuple())) + for backup in instance.list_backups( + filter_="expire_time < \"{}-{}-{}T{}:{}:{}Z\"".format( + *expire_time.timetuple())): + print(backup.name) + + # List all backups with a size greater than some bytes. + print("All backups with backup size more than 100 bytes:") + for backup in instance.list_backups(filter_="size_bytes > 100"): + print(backup.name) + + # List backups that were created after a timestamp that are also ready. + create_time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) + print("All backups created after \"{}-{}-{}T{}:{}:{}Z\" and are READY:".format( + *create_time.timetuple())) + for backup in instance.list_backups( + filter_="create_time >= \"{}-{}-{}T{}:{}:{}Z\" AND state:READY".format( + *create_time.timetuple())): + print(backup.name) + + print("All backups with pagination") + for page in instance.list_backups(page_size=2).pages: + for backup in page: + print(backup.name) +# [END spanner_list_backups] + + +# [START spanner_delete_backup] +def delete_backup(instance_id, backup_id): + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + backup = instance.backup(backup_id) + backup.reload() + + # Wait for databases that reference this backup to finish optimizing. + while backup.referencing_databases: + time.sleep(30) + backup.reload() + + # Delete the backup. + backup.delete() + + # Verify that the backup is deleted. + assert backup.exists() is False + print("Backup {} has been deleted.".format(backup.name)) +# [END spanner_delete_backup] + + +# [START spanner_update_backup] +def update_backup(instance_id, backup_id): + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + backup = instance.backup(backup_id) + backup.reload() + + # Expire time must be within 366 days of the create time of the backup. + old_expire_time = backup.expire_time + new_expire_time = old_expire_time + timedelta(days=30) + backup.update_expire_time(new_expire_time) + print("Backup {} expire time was updated from {} to {}.".format( + backup.name, old_expire_time, new_expire_time)) +# [END spanner_update_backup] + + +if __name__ == '__main__': # noqa: C901 + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + 'instance_id', help='Your Cloud Spanner instance ID.') + parser.add_argument( + '--database-id', help='Your Cloud Spanner database ID.', + default='example_db') + parser.add_argument( + '--backup-id', help='Your Cloud Spanner backup ID.', + default='example_backup') + + subparsers = parser.add_subparsers(dest='command') + subparsers.add_parser('create_backup', help=create_backup.__doc__) + subparsers.add_parser('cancel_backup', help=cancel_backup.__doc__) + subparsers.add_parser('update_backup', help=update_backup.__doc__) + subparsers.add_parser('restore_database', help=restore_database.__doc__) + subparsers.add_parser('list_backups', help=list_backups.__doc__) + subparsers.add_parser('list_backup_operations', help=list_backup_operations.__doc__) + subparsers.add_parser('list_database_operations', + help=list_database_operations.__doc__) + subparsers.add_parser('delete_backup', help=delete_backup.__doc__) + + args = parser.parse_args() + + if args.command == 'create_backup': + create_backup(args.instance_id, args.database_id, args.backup_id) + elif args.command == 'cancel_backup': + cancel_backup(args.instance_id, args.database_id, args.backup_id) + elif args.command == 'update_backup': + update_backup(args.instance_id, args.backup_id) + elif args.command == 'restore_database': + restore_database(args.instance_id, args.database_id, args.backup_id) + elif args.command == 'list_backups': + list_backups(args.instance_id, args.database_id, args.backup_id) + elif args.command == 'list_backup_operations': + list_backup_operations(args.instance_id, args.database_id) + elif args.command == 'list_database_operations': + list_database_operations(args.instance_id) + elif args.command == 'delete_backup': + delete_backup(args.instance_id, args.backup_id) + else: + print("Command {} did not match expected commands.".format(args.command)) diff --git a/spanner/cloud-client/backup_sample_test.py b/spanner/cloud-client/backup_sample_test.py new file mode 100644 index 000000000000..8ed6e8f823c2 --- /dev/null +++ b/spanner/cloud-client/backup_sample_test.py @@ -0,0 +1,116 @@ +# Copyright 2020 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import spanner +import pytest +import random +import string + +import backup_sample + + +def unique_instance_id(): + """ Creates a unique id for the database. """ + return 'test-instance-{}'.format(''.join(random.choice( + string.ascii_lowercase + string.digits) for _ in range(5))) + + +def unique_database_id(): + """ Creates a unique id for the database. """ + return 'test-db-{}'.format(''.join(random.choice( + string.ascii_lowercase + string.digits) for _ in range(5))) + + +def unique_backup_id(): + """ Creates a unique id for the backup. """ + return 'test-backup-{}'.format(''.join(random.choice( + string.ascii_lowercase + string.digits) for _ in range(5))) + + +INSTANCE_ID = unique_instance_id() +DATABASE_ID = unique_database_id() +RESTORE_DB_ID = unique_database_id() +BACKUP_ID = unique_backup_id() + + +@pytest.fixture(scope='module') +def spanner_instance(): + spanner_client = spanner.Client() + instance_config = '{}/instanceConfigs/{}'.format( + spanner_client.project_name, 'regional-us-central1') + instance = spanner_client.instance(INSTANCE_ID, instance_config) + op = instance.create() + op.result(30) # block until completion + yield instance + instance.delete() + + +@pytest.fixture(scope='module') +def database(spanner_instance): + """ Creates a temporary database that is removed after testing. """ + db = spanner_instance.database(DATABASE_ID) + db.create() + yield db + db.drop() + + +def test_create_backup(capsys, database): + backup_sample.create_backup(INSTANCE_ID, DATABASE_ID, BACKUP_ID) + out, _ = capsys.readouterr() + assert BACKUP_ID in out + + +def test_restore_database(capsys): + backup_sample.restore_database(INSTANCE_ID, RESTORE_DB_ID, BACKUP_ID) + out, _ = capsys.readouterr() + assert (DATABASE_ID + " restored to ") in out + assert (RESTORE_DB_ID + " from backup ") in out + assert BACKUP_ID in out + + +def test_list_backup_operations(capsys, spanner_instance): + backup_sample.list_backup_operations(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert BACKUP_ID in out + assert DATABASE_ID in out + + +def test_list_backups(capsys, spanner_instance): + backup_sample.list_backups(INSTANCE_ID, DATABASE_ID, BACKUP_ID) + out, _ = capsys.readouterr() + id_count = out.count(BACKUP_ID) + assert id_count == 7 + + +def test_update_backup(capsys): + backup_sample.update_backup(INSTANCE_ID, BACKUP_ID) + out, _ = capsys.readouterr() + assert BACKUP_ID in out + + +def test_delete_backup(capsys, spanner_instance): + backup_sample.delete_backup(INSTANCE_ID, BACKUP_ID) + out, _ = capsys.readouterr() + assert BACKUP_ID in out + + +def test_cancel_backup(capsys): + backup_sample.cancel_backup(INSTANCE_ID, DATABASE_ID, BACKUP_ID) + out, _ = capsys.readouterr() + cancel_success = "Backup creation was successfully cancelled." in out + cancel_failure = ( + ("Backup was created before the cancel completed." in out) and + ("Backup deleted." in out) + ) + assert cancel_success or cancel_failure