spanner: Add Cloud Spanner Backup samples (#3101)

* add backup samples

* update required spanner version

* fix lint errors

* run backup samples tests against a new instance

* fix lint

* wait for instance creation to complete

* Apply suggestions from code review

Co-Authored-By: skuruppu <[email protected]>

* add list_backups test

* fix lint

* add missing newline character in assert

* update samples to be consistent with other languages

* lint fix

* add pagination sample

* reorder tests

Co-authored-by: larkee <[email protected]>
Co-authored-by: skuruppu <[email protected]>

spanner/cloud-client/backup_sample.py

@@ -0,0 +1,281 @@
+# Copyright 2020 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This application demonstrates how to create and restore from backups
+using Cloud Spanner.
+
+For more information, see the README.rst under /spanner.
+"""
+
+import argparse
+from datetime import (
+    datetime,
+    timedelta
+)
+import time
+
+from google.cloud import spanner
+
+
+# [START spanner_create_backup]
+def create_backup(instance_id, database_id, backup_id):
+    """Creates a backup for a database."""
+    spanner_client = spanner.Client()
+    instance = spanner_client.instance(instance_id)
+    database = instance.database(database_id)
+
+    # Create a backup
+    expire_time = datetime.utcnow() + timedelta(days=14)
+    backup = instance.backup(
+        backup_id, database=database, expire_time=expire_time)
+    operation = backup.create()
+
+    # Wait for backup operation to complete.
+    operation.result()
+
+    # Verify that the backup is ready.
+    backup.reload()
+    assert backup.is_ready() is True
+
+    # Get the name, create time and backup size.
+    backup.reload()
+    print("Backup {} of size {} bytes was created at {}".format(
+        backup.name, backup.size_bytes, backup.create_time))
+# [END spanner_create_backup]
+
+
+# [START spanner_restore_database]
+def restore_database(instance_id, new_database_id, backup_id):
+    """Restores a database from a backup."""
+    spanner_client = spanner.Client()
+    instance = spanner_client.instance(instance_id)
+    # Create a backup on database_id.
+
+    # Start restoring backup to a new database.
+    backup = instance.backup(backup_id)
+    new_database = instance.database(new_database_id)
+    operation = new_database.restore(backup)
+
+    # Wait for restore operation to complete.
+    operation.result()
+
+    # Newly created database has restore information.
+    new_database.reload()
+    restore_info = new_database.restore_info
+    print("Database {} restored to {} from backup {}.".format(
+        restore_info.backup_info.source_database,
+        new_database_id,
+        restore_info.backup_info.backup))
+# [END spanner_restore_database]
+
+
+# [START spanner_cancel_backup]
+def cancel_backup(instance_id, database_id, backup_id):
+    spanner_client = spanner.Client()
+    instance = spanner_client.instance(instance_id)
+    database = instance.database(database_id)
+
+    expire_time = datetime.utcnow() + timedelta(days=30)
+
+    # Create a backup.
+    backup = instance.backup(
+        backup_id, database=database, expire_time=expire_time)
+    operation = backup.create()
+
+    # Cancel backup creation.
+    operation.cancel()
+
+    # Cancel operations are best effort so either it will complete or
+    # be cancelled.
+    while not operation.done():
+        time.sleep(300)  # 5 mins
+
+    # Deal with resource if the operation succeeded.
+    if backup.exists():
+        print("Backup was created before the cancel completed.")
+        backup.delete()
+        print("Backup deleted.")
+    else:
+        print("Backup creation was successfully cancelled.")
+# [END spanner_cancel_backup]
+
+
+# [START spanner_list_backup_operations]
+def list_backup_operations(instance_id, database_id):
+    spanner_client = spanner.Client()
+    instance = spanner_client.instance(instance_id)
+
+    # List the CreateBackup operations.
+    filter_ = (
+        "(metadata.database:{}) AND "
+        "(metadata.@type:type.googleapis.com/"
+        "google.spanner.admin.database.v1.CreateBackupMetadata)"
+    ).format(database_id)
+    operations = instance.list_backup_operations(filter_=filter_)
+    for op in operations:
+        metadata = op.metadata
+        print("Backup {} on database {}: {}% complete.".format(
+            metadata.name, metadata.database,
+            metadata.progress.progress_percent))
+# [END spanner_list_backup_operations]
+
+
+# [START spanner_list_database_operations]
+def list_database_operations(instance_id):
+    spanner_client = spanner.Client()
+    instance = spanner_client.instance(instance_id)
+
+    # List the progress of restore.
+    filter_ = (
+        "(metadata.@type:type.googleapis.com/"
+        "google.spanner.admin.database.v1.OptimizeRestoredDatabaseMetadata)"
+    )
+    operations = instance.list_database_operations(filter_=filter_)
+    for op in operations:
+        print("Database {} restored from backup is {}% optimized.".format(
+            op.metadata.name, op.metadata.progress.progress_percent))
+# [END spanner_list_database_operations]
+
+
+# [START spanner_list_backups]
+def list_backups(instance_id, database_id, backup_id):
+    spanner_client = spanner.Client()
+    instance = spanner_client.instance(instance_id)
+
+    # List all backups.
+    print("All backups:")
+    for backup in instance.list_backups():
+        print(backup.name)
+
+    # List all backups that contain a name.
+    print("All backups with backup name containing \"{}\":".format(backup_id))
+    for backup in instance.list_backups(filter_="name:{}".format(backup_id)):
+        print(backup.name)
+
+    # List all backups for a database that contains a name.
+    print("All backups with database name containing \"{}\":".format(database_id))
+    for backup in instance.list_backups(filter_="database:{}".format(database_id)):
+        print(backup.name)
+
+    # List all backups that expire before a timestamp.
+    expire_time = datetime.utcnow().replace(microsecond=0) + timedelta(days=30)
+    print("All backups with expire_time before \"{}-{}-{}T{}:{}:{}Z\":".format(
+        *expire_time.timetuple()))
+    for backup in instance.list_backups(
+            filter_="expire_time < \"{}-{}-{}T{}:{}:{}Z\"".format(
+                *expire_time.timetuple())):
+        print(backup.name)
+
+    # List all backups with a size greater than some bytes.
+    print("All backups with backup size more than 100 bytes:")
+    for backup in instance.list_backups(filter_="size_bytes > 100"):
+        print(backup.name)
+
+    # List backups that were created after a timestamp that are also ready.
+    create_time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)
+    print("All backups created after \"{}-{}-{}T{}:{}:{}Z\" and are READY:".format(
+        *create_time.timetuple()))
+    for backup in instance.list_backups(
+            filter_="create_time >= \"{}-{}-{}T{}:{}:{}Z\" AND state:READY".format(
+                *create_time.timetuple())):
+        print(backup.name)
+
+    print("All backups with pagination")
+    for page in instance.list_backups(page_size=2).pages:
+        for backup in page:
+            print(backup.name)
+# [END spanner_list_backups]
+
+
+# [START spanner_delete_backup]
+def delete_backup(instance_id, backup_id):
+    spanner_client = spanner.Client()
+    instance = spanner_client.instance(instance_id)
+    backup = instance.backup(backup_id)
+    backup.reload()
+
+    # Wait for databases that reference this backup to finish optimizing.
+    while backup.referencing_databases:
+        time.sleep(30)
+        backup.reload()
+
+    # Delete the backup.
+    backup.delete()
+
+    # Verify that the backup is deleted.
+    assert backup.exists() is False
+    print("Backup {} has been deleted.".format(backup.name))
+# [END spanner_delete_backup]
+
+
+# [START spanner_update_backup]
+def update_backup(instance_id, backup_id):
+    spanner_client = spanner.Client()
+    instance = spanner_client.instance(instance_id)
+    backup = instance.backup(backup_id)
+    backup.reload()
+
+    # Expire time must be within 366 days of the create time of the backup.
+    old_expire_time = backup.expire_time
+    new_expire_time = old_expire_time + timedelta(days=30)
+    backup.update_expire_time(new_expire_time)
+    print("Backup {} expire time was updated from {} to {}.".format(
+        backup.name, old_expire_time, new_expire_time))
+# [END spanner_update_backup]
+
+
+if __name__ == '__main__':  # noqa: C901
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument(
+        'instance_id', help='Your Cloud Spanner instance ID.')
+    parser.add_argument(
+        '--database-id', help='Your Cloud Spanner database ID.',
+        default='example_db')
+    parser.add_argument(
+        '--backup-id', help='Your Cloud Spanner backup ID.',
+        default='example_backup')
+
+    subparsers = parser.add_subparsers(dest='command')
+    subparsers.add_parser('create_backup', help=create_backup.__doc__)
+    subparsers.add_parser('cancel_backup', help=cancel_backup.__doc__)
+    subparsers.add_parser('update_backup', help=update_backup.__doc__)
+    subparsers.add_parser('restore_database', help=restore_database.__doc__)
+    subparsers.add_parser('list_backups', help=list_backups.__doc__)
+    subparsers.add_parser('list_backup_operations', help=list_backup_operations.__doc__)
+    subparsers.add_parser('list_database_operations',
+                          help=list_database_operations.__doc__)
+    subparsers.add_parser('delete_backup', help=delete_backup.__doc__)
+
+    args = parser.parse_args()
+
+    if args.command == 'create_backup':
+        create_backup(args.instance_id, args.database_id, args.backup_id)
+    elif args.command == 'cancel_backup':
+        cancel_backup(args.instance_id, args.database_id, args.backup_id)
+    elif args.command == 'update_backup':
+        update_backup(args.instance_id, args.backup_id)
+    elif args.command == 'restore_database':
+        restore_database(args.instance_id, args.database_id, args.backup_id)
+    elif args.command == 'list_backups':
+        list_backups(args.instance_id, args.database_id, args.backup_id)
+    elif args.command == 'list_backup_operations':
+        list_backup_operations(args.instance_id, args.database_id)
+    elif args.command == 'list_database_operations':
+        list_database_operations(args.instance_id)
+    elif args.command == 'delete_backup':
+        delete_backup(args.instance_id, args.backup_id)
+    else:
+        print("Command {} did not match expected commands.".format(args.command))

spanner/cloud-client/backup_sample_test.py

@@ -0,0 +1,116 @@
+# Copyright 2020 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from google.cloud import spanner
+import pytest
+import random
+import string
+
+import backup_sample
+
+
+def unique_instance_id():
+    """ Creates a unique id for the database. """
+    return 'test-instance-{}'.format(''.join(random.choice(
+        string.ascii_lowercase + string.digits) for _ in range(5)))
+
+
+def unique_database_id():
+    """ Creates a unique id for the database. """
+    return 'test-db-{}'.format(''.join(random.choice(
+        string.ascii_lowercase + string.digits) for _ in range(5)))
+
+
+def unique_backup_id():
+    """ Creates a unique id for the backup. """
+    return 'test-backup-{}'.format(''.join(random.choice(
+        string.ascii_lowercase + string.digits) for _ in range(5)))
+
+
+INSTANCE_ID = unique_instance_id()
+DATABASE_ID = unique_database_id()
+RESTORE_DB_ID = unique_database_id()
+BACKUP_ID = unique_backup_id()
+
+
+@pytest.fixture(scope='module')
+def spanner_instance():
+    spanner_client = spanner.Client()
+    instance_config = '{}/instanceConfigs/{}'.format(
+        spanner_client.project_name, 'regional-us-central1')
+    instance = spanner_client.instance(INSTANCE_ID, instance_config)
+    op = instance.create()
+    op.result(30)  # block until completion
+    yield instance
+    instance.delete()
+
+
+@pytest.fixture(scope='module')
+def database(spanner_instance):
+    """ Creates a temporary database that is removed after testing. """
+    db = spanner_instance.database(DATABASE_ID)
+    db.create()
+    yield db
+    db.drop()
+
+
+def test_create_backup(capsys, database):
+    backup_sample.create_backup(INSTANCE_ID, DATABASE_ID, BACKUP_ID)
+    out, _ = capsys.readouterr()
+    assert BACKUP_ID in out
+
+
+def test_restore_database(capsys):
+    backup_sample.restore_database(INSTANCE_ID, RESTORE_DB_ID, BACKUP_ID)
+    out, _ = capsys.readouterr()
+    assert (DATABASE_ID + " restored to ") in out
+    assert (RESTORE_DB_ID + " from backup ") in out
+    assert BACKUP_ID in out
+
+
+def test_list_backup_operations(capsys, spanner_instance):
+    backup_sample.list_backup_operations(INSTANCE_ID, DATABASE_ID)
+    out, _ = capsys.readouterr()
+    assert BACKUP_ID in out
+    assert DATABASE_ID in out
+
+
+def test_list_backups(capsys, spanner_instance):
+    backup_sample.list_backups(INSTANCE_ID, DATABASE_ID, BACKUP_ID)
+    out, _ = capsys.readouterr()
+    id_count = out.count(BACKUP_ID)
+    assert id_count == 7
+
+
+def test_update_backup(capsys):
+    backup_sample.update_backup(INSTANCE_ID, BACKUP_ID)
+    out, _ = capsys.readouterr()
+    assert BACKUP_ID in out
+
+
+def test_delete_backup(capsys, spanner_instance):
+    backup_sample.delete_backup(INSTANCE_ID, BACKUP_ID)
+    out, _ = capsys.readouterr()
+    assert BACKUP_ID in out
+
+
+def test_cancel_backup(capsys):
+    backup_sample.cancel_backup(INSTANCE_ID, DATABASE_ID, BACKUP_ID)
+    out, _ = capsys.readouterr()
+    cancel_success = "Backup creation was successfully cancelled." in out
+    cancel_failure = (
+        ("Backup was created before the cancel completed." in out) and
+        ("Backup deleted." in out)
+    )
+    assert cancel_success or cancel_failure