Download All submissions (based on #1181)

securedrop/journalist.py

@@ -578,23 +578,37 @@ def download_unread(sid):
     return bulk_download(sid, docs)
 
 
+@app.route('/download_all_unread')
+@login_required
+def download_all_unread():
+    docs = Submission.query.filter(Submission.downloaded == False).all()
+    if docs == []:
+        flash("No unread documents in collections!", "error")
+        return redirect(url_for('index'))
+    return bulk_dl('all_unread_' + g.user.username, docs)
+
+
 @app.route('/bulk', methods=('POST',))
 @login_required
 def bulk():
     action = request.form['action']
 
     doc_names_selected = request.form.getlist('doc_names_selected')
-    selected_docs = [doc for doc in g.source.collection
-                     if doc.filename in doc_names_selected]
-
+    if action == 'download_all':
+        selected_docs = g.source.collection
+    else:
+        selected_docs = [doc for doc in g.source.collection
+                         if doc.filename in doc_names_selected]
     if selected_docs == []:
         if action == 'download':
             flash("No collections selected to download!", "error")
+        elif action == 'download_all':
+            flash("No collections available to download!", "error")
         elif action == 'delete' or action == 'confirm_delete':
             flash("No collections selected to delete!", "error")
         return redirect(url_for('col', sid=g.sid))
 
-    if action == 'download':
+    if action in ('download', 'download_all'):
         return bulk_download(g.sid, selected_docs)
     elif action == 'delete':
         return bulk_delete(g.sid, selected_docs)
@@ -627,20 +641,22 @@ def bulk_delete(sid, items_selected):
 
 def bulk_download(sid, items_selected):
     source = get_source(sid)
-    filenames = [store.path(sid, item.filename) for item in items_selected]
+    return bulk_dl(source.journalist_filename, items_selected)
+
+def bulk_dl(zip_directory, items_selected):
+    filenames = [store.path(item.source.filesystem_id, item.filename)
+                 for item in items_selected]
 
     # Mark the submissions that are about to be downloaded as such
     for item in items_selected:
         if isinstance(item, Submission):
             item.downloaded = True
     db_session.commit()
 
-    zf = store.get_bulk_archive(
-        filenames,
-        zip_directory=source.journalist_filename)
+    zf = store.get_bulk_archive(filenames,
+                                zip_directory=zip_directory)
     attachment_filename = "{}--{}.zip".format(
-        source.journalist_filename,
-        datetime.utcnow().strftime("%Y-%m-%d--%H-%M-%S"))
+        zip_directory, datetime.utcnow().strftime("%Y-%m-%d--%H-%M-%S"))
     return send_file(zf.name, mimetype="application/zip",
                      attachment_filename=attachment_filename,
                      as_attachment=True)

securedrop/journalist_templates/col.html

@@ -21,11 +21,12 @@
   {% if source.collection %}
     <p>The documents are stored encrypted for security. To read them, you will need to decrypt them using GPG.</p>
     <form action="/bulk" method="post">
-      <div class="document-actions">
-        <div id='select-container'></div>
-        <button type="submit" name="action" value="download"><i class="fa fa-download"></i> Download selected</button>
-        <button type="submit" name="action" value="confirm_delete" class="danger" id="delete_selected"><i class="fa fa-times"></i> Delete selected</button>
-      </div>
+    <div class="document-actions">
+      <div id='select-container'></div>
+      <button type="submit" name="action" value="download_all"><i class="fa fa-download"></i> Download all</button>
+      <button type="submit" name="action" value="download"><i class="fa fa-download"></i> Download selected</button>
+      <button type="submit" name="action" value="confirm_delete" class="danger" id="delete_selected"><i class="fa fa-times"></i> Delete selected</button>
+    </div>
       <ul id="submissions" class="plain submissions">
         {% for doc in source.collection %}
           <li class="submission"><input type="checkbox" name="doc_names_selected" value="{{ doc.filename }}" class="doc-check"/>

securedrop/journalist_templates/index.html

@@ -8,6 +8,7 @@ <h2><span class="headline">Sources</span></h2>
       <input name="csrf_token" type="hidden" value="{{ csrf_token() }}"/>
       <p>
         <span id="select_all" class="select"><i class="fa fa-check-square-o"></i> select all</span> <span id="select_none" class="select"><i class="fa fa-square-o"></i> select none</span>
+        <a href="{{ url_for('download_all_unread') }}" class="btn small">Download all unread</a>
         <button type="submit" id="delete_collections" name="action" value="delete" class="small"><i class="fa fa-minus-circle"></i> Delete selected</button>
         <button type="submit" name="action" value="star" class="small"><i class="fa fa-minus-circle"></i> Star Selected</button>
         <button type="submit" name="action" value="un-star" class="small"><i class="fa fa-minus-circle"></i> Un-star Selected</button>

securedrop/tests/test_unit_journalist.py

@@ -7,7 +7,7 @@
 import mock
 
 from flask_testing import TestCase
-from flask import url_for, escape
+from flask import url_for, escape, g
 
 import crypto_util
 import journalist
@@ -415,27 +415,149 @@ def test_edit_hotp(self):
         # should redirect to verification page
         self.assertRedirects(res, url_for('account_new_two_factor'))
 
-    def test_bulk_download(self):
+    def test_selected_bulk_download(self):
         sid = 'EQZGCJBRGISGOTC2NZVWG6LILJBHEV3CINNEWSCLLFTUWZJPKJFECLS2NZ4G4U3QOZCFKTTPNZMVIWDCJBBHMUDBGFHXCQ3R'
         source = Source(sid, crypto_util.display_id())
         db_session.add(source)
         db_session.commit()
-        files = ['1-abc1-msg.gpg', '2-abc2-msg.gpg']
-        filenames = common.setup_test_docs(sid, files)
+        files = ['1-abc1-msg.gpg', '2-abc2-msg.gpg', '3-abc3-msg.gpg', '4-abc4-msg.gpg']
+        selected_files = files[:2]
+        unselected_files = files[2:]
+        common.setup_test_docs(sid, files)
 
         self._login_user()
         rv = self.client.post('/bulk', data=dict(
             action='download',
             sid=sid,
-            doc_names_selected=files
+            doc_names_selected=selected_files
         ))
 
         self.assertEqual(rv.status_code, 200)
         self.assertEqual(rv.content_type, 'application/zip')
         self.assertTrue(zipfile.is_zipfile(StringIO(rv.data)))
-        self.assertTrue(zipfile.ZipFile(StringIO(rv.data)).getinfo(
-            os.path.join(source.journalist_filename, files[0])
+
+        for file in selected_files:
+            self.assertTrue(zipfile.ZipFile(StringIO(rv.data)).getinfo(
+                 os.path.join(source.journalist_filename, file)
+            ))
+
+        for file in unselected_files:
+            try:
+                zipfile.ZipFile(StringIO(rv.data)).getinfo(
+                    os.path.join(source.journalist_filename, file))
+            except KeyError:
+                pass
+            else:
+                self.assertTrue(False)
+
+    def test_download_all_bulk_download(self):
+        sid = 'EQZGCJBRGISGOTC2NZVWG6LILJBHEV3CINNEWSCLLFTUWZJPKJFECLS2NZ4G4U3QOZCFKTTPNZMVIWDCJBBHMUDBGFHXCQ3R'
+        source = Source(sid, crypto_util.display_id())
+        db_session.add(source)
+        db_session.commit()
+        files = ['1-abc1-msg.gpg', '2-abc2-msg.gpg', '3-abc3-msg.gpg', '4-abc4-msg.gpg']
+        common.setup_test_docs(sid, files)
+
+        self._login_user()
+        rv = self.client.post('/bulk', data=dict(
+             action='download_all',
+             sid=sid
+        ))
+
+        self.assertEqual(rv.status_code, 200)
+        self.assertEqual(rv.content_type, 'application/zip')
+        self.assertTrue(zipfile.is_zipfile(StringIO(rv.data)))
+        for file in files:
+            self.assertTrue(zipfile.ZipFile(StringIO(rv.data)).getinfo(
+                 os.path.join(source.journalist_filename, file)
+            ))
+
+    def test_download_bulk_download_nothing_selected(self):
+        sid = 'EQZGCJBRGISGOTC2NZVWG6LILJBHEV3CINNEWSCLLFTUWZJPKJFECLS2NZ4G4U3QOZCFKTTPNZMVIWDCJBBHMUDBGFHXCQ3R'
+        source = Source(sid, crypto_util.display_id())
+        db_session.add(source)
+        db_session.commit()
+        files = ['1-abc1-msg.gpg', '2-abc2-msg.gpg', '3-abc3-msg.gpg', '4-abc4-msg.gpg']
+        common.setup_test_docs(sid, files)
+
+        self._login_user()
+        rv = self.client.post('/bulk', data=dict(
+                action='download',
+                sid=sid,
+                doc_names_selected = []
+        ))
+        self.assertRedirects(rv, url_for('col', sid=sid))
+
+    def test_download_bulk_download_all_no_submissions(self):
+        sid = 'EQZGCJBRGISGOTC2NZVWG6LILJBHEV3CINNEWSCLLFTUWZJPKJFECLS2NZ4G4U3QOZCFKTTPNZMVIWDCJBBHMUDBGFHXCQ3R'
+        source = Source(sid, crypto_util.display_id())
+        db_session.add(source)
+        db_session.commit()
+
+        self._login_user()
+        rv = self.client.post('/bulk', data=dict(
+                action='download_all',
+                sid=sid,
+        ))
+        self.assertRedirects(rv, url_for('col', sid=sid))
+
+    def test_delete_bulk_delete_nothing_selected(self):
+        sid = 'EQZGCJBRGISGOTC2NZVWG6LILJBHEV3CINNEWSCLLFTUWZJPKJFECLS2NZ4G4U3QOZCFKTTPNZMVIWDCJBBHMUDBGFHXCQ3R'
+        source = Source(sid, crypto_util.display_id())
+        db_session.add(source)
+        db_session.commit()
+        files = ['1-abc1-msg.gpg', '2-abc2-msg.gpg', '3-abc3-msg.gpg', '4-abc4-msg.gpg']
+        common.setup_test_docs(sid, files)
+
+        self._login_user()
+        rv = self.client.post('/bulk', data=dict(
+                action='delete',
+                sid=sid,
+                doc_names_selected = []
+        ))
+        self.assertRedirects(rv, url_for('col', sid=sid))
+
+    def test_download_all(self):
+        sid = 'EQZGCJBRGISGOTC2NZVWG6LILJBHEV3CINNEWSCLLFTUWZJPKJFECLS2NZ4G4U3QOZCFKTTPNZMVIWDCJBBHMUDBGFHXCQ3R'
+        source = Source(sid, crypto_util.display_id())
+        db_session.add(source)
+        db_session.commit()
+        files = ['1-abc1-msg.gpg', '2-abc2-msg.gpg', '3-abc3-msg.gpg', '4-abc4-msg.gpg']
+        selected_files = files[::2]
+        unselected_files = files[1::2]
+        common.setup_test_docs(sid, files)
+
+        self._login_user()
+        rv = self.client.post('/bulk', data=dict(
+                action='download',
+                sid=sid,
+                doc_names_selected=selected_files
+        ))
+        rv = self.client.get('/download_all_unread')
+        self.assertEqual(rv.status_code, 200)
+        self.assertEqual(rv.content_type, 'application/zip')
+        self.assertTrue(len(zipfile.ZipFile(StringIO(rv.data)).namelist()) == \
+                        len(unselected_files))
+        for file in unselected_files:
+            self.assertTrue(zipfile.ZipFile(StringIO(rv.data)).getinfo(
+                os.path.join('all_unread_'+g.user.username, file
+            )))
+
+    def test_download_all_with_no_unread_submissions(self):
+        sid = 'EQZGCJBRGISGOTC2NZVWG6LILJBHEV3CINNEWSCLLFTUWZJPKJFECLS2NZ4G4U3QOZCFKTTPNZMVIWDCJBBHMUDBGFHXCQ3R'
+        source = Source(sid, crypto_util.display_id())
+        db_session.add(source)
+        db_session.commit()
+        files = ['1-abc1-msg.gpg', '2-abc2-msg.gpg', '3-abc3-msg.gpg', '4-abc4-msg.gpg']
+        common.setup_test_docs(sid, files)
+
+        self._login_user()
+        rv = self.client.post('/bulk', data=dict(
+                action='download_all',
+                sid=sid,
         ))
+        rv = self.client.get('/download_all_unread')
+        self.assertRedirects(rv, url_for('index'))
 
     def test_max_password_length(self):
         """Creating a Journalist with a password that is greater than the