From 7a02e5ec96d24d3419328a3dab721e67fa923f35 Mon Sep 17 00:00:00 2001 From: Louis-Dominique Dubeau Date: Wed, 3 Apr 2019 10:51:41 -0400 Subject: [PATCH] Add the sync-state command. --- README.rst | 42 +++++++++++++++++++++++++++++- VERSION | 2 +- btw_backup/__main__.py | 58 +++++++++++++++++++++++++++++------------- btw_backup/sync.py | 18 ++++++++++--- tests.py | 57 ++++++++++++++++++++++++++++++++++++++--- 5 files changed, 151 insertions(+), 26 deletions(-) diff --git a/README.rst b/README.rst index c2269ae..a8752c3 100644 --- a/README.rst +++ b/README.rst @@ -9,7 +9,7 @@ Synopsis :: usage: btw-backup [-h] [-q] [--config-dir CONFIG_DIR] [--version] - {fs,fs-init,list,db,sync} ... + {fs,fs-init,list,db,sync,sync-state} ... optional arguments: -h, --help show this help message and exit @@ -32,6 +32,14 @@ Synopsis immediately. Synchronizations happen automatically after a backup is run. +.. note:: The ``sync-state`` command is equally unusual. It is meant to verify + and manipulate the current sync state. + +Requisites +========== + +* ``apt-get install rdiff-backup`` + Configuration ============= @@ -246,6 +254,23 @@ A: No, there is still a problem. See, ``rdiff-backup`` says it The script ``misc/script.sh`` illustrates the issue. +Checking and Resetting the Sync State +===================================== + +As mentioned above, ``btw-backup`` records a sync state that can allow it to +recover from being forcibly interrupted during a sync. Over the years this file +can grow quite a bit. The ``sync-state`` subcommand allows checking and +resetting this file. + +You can use ``btw-backup sync-state --list`` to list the files that have not +been synced. + +You can use ``btw-backup sync-state --reset`` to reset the state to empty. This +command will fail if there are any files that have not been synced. + +.. warning:: ``sync-state --reset`` is not bullet-proof. Use your judgment as to + when to run it. + Security ======== @@ -263,5 +288,20 @@ ftp site.) By default, ``btw-backup`` invokes ``s3cmd`` with ``--server-side-encryption``, which encrypts the data on the S3 server. +Testing +======= + +* See ``Requisites`` above. + +* Create a virtual environment for this purpose. + +* Activate the virtual environment. + +* ``pip install -e .`` + +* ``npm install`` + +* ``python setup.py nosetest`` + .. LocalWords: btw hoc fs init subcommands py globals config src .. LocalWords: rdiff pytimeparse UID GID dst tarfile txt dumpall diff --git a/VERSION b/VERSION index 9084fa2..26aaba0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.0 +1.2.0 diff --git a/btw_backup/__main__.py b/btw_backup/__main__.py index 5cd59c0..f68f6a4 100644 --- a/btw_backup/__main__.py +++ b/btw_backup/__main__.py @@ -303,6 +303,7 @@ def sync_path(self, path): class Sync(BaseBackupCommand): + def __init__(self, args): """ Sync ROOT_PATH with S3 storage. You would use this if you already @@ -310,13 +311,26 @@ def __init__(self, args): to a new S3 location. """ super(Sync, self).__init__(args) - if self.args.full and self.args.list: - raise Exception("--full and --list are not allowed together") - self.suppress_sync = False - def sync(self): - if not self.suppress_sync: - return super(Sync, self).sync() + def execute(self): + if self.args.full: + self.sync_path("") + + # We do not execute self.sync because this is done by default + # with all command executions. + +class SyncStateCommand(Command): + + def __init__(self, args): + """ + Obtain information or manipulate the sync state. + """ + super(SyncStateCommand, self).__init__(args) + if self.args.list and self.args.reset: + fatal("--list and --reset are not allowed together") + if not (self.args.list or self.args.reset): + fatal("must specify --list or --reset") + self.sync_state = SyncState(self.sync_state_path) def execute(self): if self.args.list: @@ -326,12 +340,10 @@ def execute(self): for path in current_state["push"]: print("Must push: " + path) - self.suppress_sync = True - elif self.args.full: - self.sync_path("") - - # We do not execute self.sync because this is done by default - # with all command executions. + elif self.args.reset: + current_state = self.sync_state.current_state + self.sync_state.reset() + print("The state was reset") class RdiffBackupCommand(BaseBackupCommand): @@ -893,7 +905,6 @@ def main(): db_sp.add_argument('--fake-dumpall', action='store', help=argparse.SUPPRESS) - sync_sp = subparsers.add_parser( "sync", description=Sync.__doc__, @@ -906,10 +917,23 @@ def main(): help="do a full sync of everything in ROOT_PATH to " "S3 storage; the default is to sync only files " "needing syncing") - sync_sp.add_argument("--list", - dest="list", - action="store_true", - help="only list the files that need syncing") + + sync_state_sp = subparsers.add_parser( + "sync-state", + description=SyncStateCommand.__doc__, + help="get information about the sync state or manipulate it", + formatter_class=argparse.RawTextHelpFormatter) + sync_state_sp.set_defaults(class_=SyncStateCommand) + + sync_state_sp.add_argument("--list", + dest="list", + action="store_true", + help="only list the files that need syncing") + + sync_state_sp.add_argument("--reset", + dest="reset", + action="store_true", + help="reset the state, if it is safe to do so") try: try: diff --git a/btw_backup/sync.py b/btw_backup/sync.py index c462c04..abe936e 100644 --- a/btw_backup/sync.py +++ b/btw_backup/sync.py @@ -7,7 +7,7 @@ import sys import traceback -from .errors import ImproperlyConfigured +from .errors import ImproperlyConfigured, FatalUserError def format_exception(): return "".join(traceback.format_exception(*sys.exc_info())) @@ -135,8 +135,18 @@ def sync_done(self, path): """ self._update_state("-sync", path) + def reset(self): + """ + """ + if len(self.current_state["sync"]) or len(self.current_state["push"]): + raise FatalUserError( + "cannot reset: some files must be synced or pushed") + self._file.seek(0) + self._file.truncate() + class S3(object): + def __init__(self, general_config, state): self.s3_uri_prefix = general_config.get("S3_URI_PREFIX") if self.s3_uri_prefix is None: @@ -184,7 +194,7 @@ def run(self): for to_push in set(current["push"]): try: self._push(to_push) - except: # pylint: disable=bare-except + except: # pylint: disable=bare-except stderr = self._stderr print("Error while processing: " + to_push, file=stderr) print(format_exception(), file=stderr) @@ -192,7 +202,7 @@ def run(self): for to_sync in set(current["sync"]): try: self._sync(to_sync) - except: # pylint: disable=bare-except + except: # pylint: disable=bare-except stderr = self._stderr print("Error while processing: " + to_sync, file=stderr) print(format_exception(), file=stderr) @@ -236,6 +246,7 @@ def _stderr(self): class AWSCliS3(S3): + def __init__(self, general_config, state): super(AWSCliS3, self).__init__(general_config, state) @@ -278,6 +289,7 @@ def _do_push(self, path): stdout=stdout, stderr=stdout) class S3Cmd(S3): + def __init__(self, general_config, state): super(S3Cmd, self).__init__(general_config, state) diff --git a/tests.py b/tests.py index 0b68c86..53823e8 100644 --- a/tests.py +++ b/tests.py @@ -1111,6 +1111,35 @@ def test_full(self): self.assertNoError(Backup(["sync", "--full"])) + def test_default(self): + # Create some files in dst + shutil.copytree(self.src, self.dst_full) + + # Write a fake state. + with open(os.path.join(config_dir, "sync_state"), 'w') as state: + state.write("""\ +2016-01-01T12:00:00 +push dst/a +""") + + self.assertNoError(Backup(["sync"]), dont_compare=True) + + self.assertTrue(exists_on_server("foo/backups/dst/a")) + + +class SyncStateCommandTest(BackupTestMixin, unittest.TestCase): + src = os.path.join(os.getcwd(), "test-data/src") + + def setUp(self): + self.dst = "dst" + self.dst_full = os.path.join(root_dir, self.dst) + + def tearDown(self): + if os.path.exists(self.dst_full): + shutil.rmtree(self.dst_full) + reset_tmpdir() + reset_server() + super(SyncStateCommandTest, self).tearDown() + def test_list(self): # Create some files in dst shutil.copytree(self.src, self.dst_full) @@ -1123,13 +1152,13 @@ def test_list(self): 2016-01-01T12:00:00 +sync dst """) - self.assertNoError(Backup(["sync", "--list"]), """\ + self.assertNoError(Backup(["sync-state", "--list"]), """\ Must sync: dst Must push: dst/a Must push: dst/b\ """) - def test_default(self): + def test_reset_fails(self): # Create some files in dst shutil.copytree(self.src, self.dst_full) @@ -1137,11 +1166,31 @@ def test_default(self): with open(os.path.join(config_dir, "sync_state"), 'w') as state: state.write("""\ 2016-01-01T12:00:00 +push dst/a +2016-01-01T12:00:00 +push dst/b +2016-01-01T12:00:00 +sync dst """) - self.assertNoError(Backup(["sync"]), dont_compare=True) + self.assertError(Backup(["sync-state", "--reset"]), + "btw_backup: cannot reset: some files " + "must be synced or pushed", 1) - self.assertTrue(exists_on_server("foo/backups/dst/a")) + def test_reset_works(self): + # Create some files in dst + shutil.copytree(self.src, self.dst_full) + + # Write a fake state. + with open(os.path.join(config_dir, "sync_state"), 'w') as state: + state.write("""\ +2016-01-01T12:00:00 +push dst/a +2016-01-01T12:00:00 +push dst/b +2016-01-01T12:00:00 +sync dst +2016-01-01T12:00:00 -push dst/a +2016-01-01T12:00:00 -push dst/b +2016-01-01T12:00:00 -sync dst +""") + + self.assertNoError(Backup(["sync-state", "--reset"]), + "The state was reset") class CommonDB(BackupTestMixin):