Add the sync-state command.

README.rst

@@ -9,7 +9,7 @@ Synopsis
 ::
 
     usage: btw-backup [-h] [-q] [--config-dir CONFIG_DIR] [--version]
-                      {fs,fs-init,list,db,sync} ...
+                      {fs,fs-init,list,db,sync,sync-state} ...
 
     optional arguments:
       -h, --help            show this help message and exit
@@ -32,6 +32,14 @@ Synopsis
           immediately. Synchronizations happen automatically after a
           backup is run.
 
+.. note:: The ``sync-state`` command is equally unusual. It is meant to verify
+          and manipulate the current sync state.
+
+Requisites
+==========
+
+* ``apt-get install rdiff-backup``
+
 Configuration
 =============
 
@@ -246,6 +254,23 @@ A: No, there is still a problem. See, ``rdiff-backup`` says it
 
    The script ``misc/script.sh`` illustrates the issue.
 
+Checking and Resetting the Sync State
+=====================================
+
+As mentioned above, ``btw-backup`` records a sync state that can allow it to
+recover from being forcibly interrupted during a sync. Over the years this file
+can grow quite a bit. The ``sync-state`` subcommand allows checking and
+resetting this file.
+
+You can use ``btw-backup sync-state --list`` to list the files that have not
+been synced.
+
+You can use ``btw-backup sync-state --reset`` to reset the state to empty. This
+command will fail if there are any files that have not been synced.
+
+.. warning:: ``sync-state --reset`` is not bullet-proof. Use your judgment as to
+             when to run it.
+
 Security
 ========
 
@@ -263,5 +288,20 @@ ftp site.) By default, ``btw-backup`` invokes ``s3cmd`` with
 ``--server-side-encryption``, which encrypts the data on the S3
 server.
 
+Testing
+=======
+
+* See ``Requisites`` above.
+
+* Create a virtual environment for this purpose.
+
+* Activate the virtual environment.
+
+* ``pip install -e .``
+
+* ``npm install``
+
+* ``python setup.py nosetest``
+
 ..  LocalWords:  btw hoc fs init subcommands py globals config src
 ..  LocalWords:  rdiff pytimeparse UID GID dst tarfile txt dumpall

VERSION

@@ -1 +1 @@
-1.1.0
+1.2.0

btw_backup/__main__.py

@@ -303,20 +303,34 @@ def sync_path(self, path):
 
 
 class Sync(BaseBackupCommand):
+
     def __init__(self, args):
         """
         Sync ROOT_PATH with S3 storage. You would use this if you already
         have backups on the local machine and want to synchronize them
         to a new S3 location.
         """
         super(Sync, self).__init__(args)
-        if self.args.full and self.args.list:
-            raise Exception("--full and --list are not allowed together")
-        self.suppress_sync = False
 
-    def sync(self):
-        if not self.suppress_sync:
-            return super(Sync, self).sync()
+    def execute(self):
+        if self.args.full:
+            self.sync_path("")
+
+        # We do not execute self.sync because this is done by default
+        # with all command executions.
+
+class SyncStateCommand(Command):
+
+    def __init__(self, args):
+        """
+        Obtain information or manipulate the sync state.
+        """
+        super(SyncStateCommand, self).__init__(args)
+        if self.args.list and self.args.reset:
+            fatal("--list and --reset are not allowed together")
+        if not (self.args.list or self.args.reset):
+            fatal("must specify --list or --reset")
+        self.sync_state = SyncState(self.sync_state_path)
 
     def execute(self):
         if self.args.list:
@@ -326,12 +340,10 @@ def execute(self):
 
             for path in current_state["push"]:
                 print("Must push: " + path)
-            self.suppress_sync = True
-        elif self.args.full:
-            self.sync_path("")
-
-        # We do not execute self.sync because this is done by default
-        # with all command executions.
+        elif self.args.reset:
+            current_state = self.sync_state.current_state
+            self.sync_state.reset()
+            print("The state was reset")
 
 class RdiffBackupCommand(BaseBackupCommand):
 
@@ -893,7 +905,6 @@ def main():
     db_sp.add_argument('--fake-dumpall', action='store',
                        help=argparse.SUPPRESS)
 
-
     sync_sp = subparsers.add_parser(
         "sync",
         description=Sync.__doc__,
@@ -906,10 +917,23 @@ def main():
                          help="do a full sync of everything in ROOT_PATH to "
                          "S3 storage; the default is to sync only files "
                          "needing syncing")
-    sync_sp.add_argument("--list",
-                         dest="list",
-                         action="store_true",
-                         help="only list the files that need syncing")
+
+    sync_state_sp = subparsers.add_parser(
+        "sync-state",
+        description=SyncStateCommand.__doc__,
+        help="get information about the sync state or manipulate it",
+        formatter_class=argparse.RawTextHelpFormatter)
+    sync_state_sp.set_defaults(class_=SyncStateCommand)
+
+    sync_state_sp.add_argument("--list",
+                               dest="list",
+                               action="store_true",
+                               help="only list the files that need syncing")
+
+    sync_state_sp.add_argument("--reset",
+                               dest="reset",
+                               action="store_true",
+                               help="reset the state, if it is safe to do so")
 
     try:
         try:

btw_backup/sync.py

@@ -7,7 +7,7 @@
 import sys
 import traceback
 
-from .errors import ImproperlyConfigured
+from .errors import ImproperlyConfigured, FatalUserError
 
 def format_exception():
     return "".join(traceback.format_exception(*sys.exc_info()))
@@ -135,8 +135,18 @@ def sync_done(self, path):
         """
         self._update_state("-sync", path)
 
+    def reset(self):
+        """
+        """
+        if len(self.current_state["sync"]) or len(self.current_state["push"]):
+            raise FatalUserError(
+                "cannot reset: some files must be synced or pushed")
+        self._file.seek(0)
+        self._file.truncate()
+
 
 class S3(object):
+
     def __init__(self, general_config, state):
         self.s3_uri_prefix = general_config.get("S3_URI_PREFIX")
         if self.s3_uri_prefix is None:
@@ -184,15 +194,15 @@ def run(self):
         for to_push in set(current["push"]):
             try:
                 self._push(to_push)
-            except: # pylint: disable=bare-except
+            except:  # pylint: disable=bare-except
                 stderr = self._stderr
                 print("Error while processing: " + to_push, file=stderr)
                 print(format_exception(), file=stderr)
 
         for to_sync in set(current["sync"]):
             try:
                 self._sync(to_sync)
-            except: # pylint: disable=bare-except
+            except:  # pylint: disable=bare-except
                 stderr = self._stderr
                 print("Error while processing: " + to_sync, file=stderr)
                 print(format_exception(), file=stderr)
@@ -236,6 +246,7 @@ def _stderr(self):
 
 
 class AWSCliS3(S3):
+
     def __init__(self, general_config, state):
         super(AWSCliS3, self).__init__(general_config, state)
 
@@ -278,6 +289,7 @@ def _do_push(self, path):
                               stdout=stdout, stderr=stdout)
 
 class S3Cmd(S3):
+
     def __init__(self, general_config, state):
         super(S3Cmd, self).__init__(general_config, state)
 

tests.py

@@ -1111,6 +1111,35 @@ def test_full(self):
 
         self.assertNoError(Backup(["sync", "--full"]))
 
+    def test_default(self):
+        # Create some files in dst
+        shutil.copytree(self.src, self.dst_full)
+
+        # Write a fake state.
+        with open(os.path.join(config_dir, "sync_state"), 'w') as state:
+            state.write("""\
+2016-01-01T12:00:00 +push dst/a
+""")
+
+        self.assertNoError(Backup(["sync"]), dont_compare=True)
+
+        self.assertTrue(exists_on_server("foo/backups/dst/a"))
+
+
+class SyncStateCommandTest(BackupTestMixin, unittest.TestCase):
+    src = os.path.join(os.getcwd(), "test-data/src")
+
+    def setUp(self):
+        self.dst = "dst"
+        self.dst_full = os.path.join(root_dir, self.dst)
+
+    def tearDown(self):
+        if os.path.exists(self.dst_full):
+            shutil.rmtree(self.dst_full)
+        reset_tmpdir()
+        reset_server()
+        super(SyncStateCommandTest, self).tearDown()
+
     def test_list(self):
         # Create some files in dst
         shutil.copytree(self.src, self.dst_full)
@@ -1123,25 +1152,45 @@ def test_list(self):
 2016-01-01T12:00:00 +sync dst
 """)
 
-        self.assertNoError(Backup(["sync", "--list"]), """\
+        self.assertNoError(Backup(["sync-state", "--list"]), """\
 Must sync: dst
 Must push: dst/a
 Must push: dst/b\
 """)
 
-    def test_default(self):
+    def test_reset_fails(self):
         # Create some files in dst
         shutil.copytree(self.src, self.dst_full)
 
         # Write a fake state.
         with open(os.path.join(config_dir, "sync_state"), 'w') as state:
             state.write("""\
 2016-01-01T12:00:00 +push dst/a
+2016-01-01T12:00:00 +push dst/b
+2016-01-01T12:00:00 +sync dst
 """)
 
-        self.assertNoError(Backup(["sync"]), dont_compare=True)
+        self.assertError(Backup(["sync-state", "--reset"]),
+                         "btw_backup: cannot reset: some files "
+                         "must be synced or pushed", 1)
 
-        self.assertTrue(exists_on_server("foo/backups/dst/a"))
+    def test_reset_works(self):
+        # Create some files in dst
+        shutil.copytree(self.src, self.dst_full)
+
+        # Write a fake state.
+        with open(os.path.join(config_dir, "sync_state"), 'w') as state:
+            state.write("""\
+2016-01-01T12:00:00 +push dst/a
+2016-01-01T12:00:00 +push dst/b
+2016-01-01T12:00:00 +sync dst
+2016-01-01T12:00:00 -push dst/a
+2016-01-01T12:00:00 -push dst/b
+2016-01-01T12:00:00 -sync dst
+""")
+
+        self.assertNoError(Backup(["sync-state", "--reset"]),
+                           "The state was reset")
 
 
 class CommonDB(BackupTestMixin):