Download unread/ all selected sources

securedrop/journalist.py

@@ -459,8 +459,11 @@ def delete_collection(source_id):
 @app.route('/col/process', methods=('POST',))
 @login_required
 def col_process():
-    actions = {'delete': col_delete, 'star': col_star, 'un-star': col_un_star}
+    actions = {'download-unread': col_download_unread,
+               'download-all': col_download_all, 'star': col_star,
+               'un-star': col_un_star, 'delete': col_delete}
     if 'cols_selected' not in request.form:
+        flash('No collections selected!', 'error')
         return redirect(url_for('index'))
 
     # getlist is cgi.FieldStorage.getlist
@@ -474,6 +477,28 @@ def col_process():
     return method(cols_selected)
 
 
+def col_download_unread(cols_selected):
+    """Download all unread submissions from all selected sources."""
+    submissions = []
+    for sid in cols_selected:
+        id = Source.query.filter(Source.filesystem_id == sid).one().id
+        submissions += Submission.query.filter(Submission.downloaded == False,
+                                               Submission.source_id == id).all()
+    if submissions == []:
+        flash("No unread submissions in collections selected!", "error")
+        return redirect(url_for('index'))
+    return download("unread", submissions)
+
+
+def col_download_all(cols_selected):
+    """Download all submissions from all selected sources."""
+    submissions = []
+    for sid in cols_selected:
+        id = Source.query.filter(Source.filesystem_id == sid).one().id
+        submissions += Submission.query.filter(Submission.source_id == id).all()
+    return download("all", submissions)
+
+
 def col_star(cols_selected):
     for sid in cols_selected:
         make_star_true(sid)
@@ -519,15 +544,18 @@ def col_delete(cols_selected):
 
 @app.route('/col/<sid>/<fn>')
 @login_required
-def doc(sid, fn):
+def download_single_submission(sid, fn):
+    """Sends a client the contents of a single submission."""
     if '..' in fn or fn.startswith('/'):
         abort(404)
+
     try:
         Submission.query.filter(
             Submission.filename == fn).one().downloaded = True
+        db_session.commit()
     except NoResultFound as e:
         app.logger.error("Could not mark " + fn + " as downloaded: %s" % (e,))
-    db_session.commit()
+
     return send_file(store.path(sid, fn), mimetype="application/pgp-encrypted")
 
 
@@ -571,12 +599,15 @@ def generate_code():
 
 @app.route('/download_unread/<sid>')
 @login_required
-def download_unread(sid):
+def download_unread_sid(sid):
     id = Source.query.filter(Source.filesystem_id == sid).one().id
-    docs = Submission.query.filter(
-        Submission.source_id == id,
-        Submission.downloaded == False).all()
-    return bulk_download(sid, docs)
+    submissions = Submission.query.filter(Submission.source_id == id,
+                                          Submission.downloaded == False).all()
+    if submissions == []:
+        flash("No unread submissions for this source!")
+        return redirect(url_for('col', sid=sid))
+    source = get_source(sid)
+    return download(source.journalist_filename, submissions)
 
 
 @app.route('/bulk', methods=('POST',))
@@ -587,16 +618,16 @@ def bulk():
     doc_names_selected = request.form.getlist('doc_names_selected')
     selected_docs = [doc for doc in g.source.collection
                      if doc.filename in doc_names_selected]
-
     if selected_docs == []:
         if action == 'download':
             flash("No collections selected to download!", "error")
-        elif action == 'delete' or action == 'confirm_delete':
+        elif action in ('delete', 'confirm_delete'):
             flash("No collections selected to delete!", "error")
         return redirect(url_for('col', sid=g.sid))
 
     if action == 'download':
-        return bulk_download(g.sid, selected_docs)
+        source = get_source(g.sid)
+        return download(source.journalist_filename, selected_docs)
     elif action == 'delete':
         return bulk_delete(g.sid, selected_docs)
     elif action == 'confirm_delete':
@@ -626,22 +657,30 @@ def bulk_delete(sid, items_selected):
     return redirect(url_for('col', sid=sid))
 
 
-def bulk_download(sid, items_selected):
-    source = get_source(sid)
-    filenames = [store.path(sid, item.filename) for item in items_selected]
+def download(zip_basename, submissions):
+    """Send client contents of zipfile *zip_basename*-<timestamp>.zip
+    containing *submissions*. The zipfile, being a
+    :class:`tempfile.NamedTemporaryFile`, is stored on disk only
+    temporarily.
+
+    :param str zip_basename: The basename of the zipfile download.
 
+    :param list submissions: A list of :class:`db.Submission`s to 
+                             include in the zipfile.
+    """
     # Mark the submissions that are about to be downloaded as such
-    for item in items_selected:
-        if isinstance(item, Submission):
-            item.downloaded = True
+    for submission in submissions:
+        submission.downloaded = True
     db_session.commit()
 
-    zf = store.get_bulk_archive(
-        filenames,
-        zip_directory=source.journalist_filename)
+    filenames = [store.path(submission.source.filesystem_id,
+                            submission.filename)
+                 for submission in submissions]
+
+    zf = store.get_bulk_archive(filenames,
+                                zip_directory=zip_basename)
     attachment_filename = "{}--{}.zip".format(
-        source.journalist_filename,
-        datetime.utcnow().strftime("%Y-%m-%d--%H-%M-%S"))
+        zip_basename, datetime.utcnow().strftime("%Y-%m-%d--%H-%M-%S"))
     return send_file(zf.name, mimetype="application/zip",
                      attachment_filename=attachment_filename,
                      as_attachment=True)

securedrop/journalist_templates/col.html

@@ -23,8 +23,8 @@
     <form action="/bulk" method="post">
       <div class="document-actions">
         <div id='select-container'></div>
-        <button type="submit" name="action" value="download"><i class="fa fa-download"></i> Download selected</button>
-        <button type="submit" name="action" value="confirm_delete" class="danger" id="delete_selected"><i class="fa fa-times"></i> Delete selected</button>
+        <button type="submit" name="action" value="download"><i class="fa fa-download"></i> Download</button>
+        <button type="submit" name="action" value="confirm_delete" class="danger" id="delete_selected"><i class="fa fa-trash-o"></i> Delete</button>
       </div>
       <ul id="submissions" class="plain submissions">
         {% for doc in source.collection %}
@@ -85,7 +85,7 @@ <h3><i class="fa fa-reply"></i> Reply</h3>
     <input name="csrf_token" type="hidden" value="{{ csrf_token() }}"/>
     <input type="hidden" name="sid" value="{{ sid }}"/>
     <input type="hidden" name="col_name" value="{{ source.journalist_designation }}"/>
-    <button type="submit" class="danger"><i class="fa fa-times"></i> Delete collection</button>
+    <button type="submit" class="danger"><i class="fa fa-trash-o"></i> Delete Collection</button>
   </form>
 
 </div>

securedrop/journalist_templates/index.html

@@ -8,9 +8,11 @@ <h2><span class="headline">Sources</span></h2>
       <input name="csrf_token" type="hidden" value="{{ csrf_token() }}"/>
       <p>
         <span id="select_all" class="select"><i class="fa fa-check-square-o"></i> select all</span> <span id="select_none" class="select"><i class="fa fa-square-o"></i> select none</span>
-        <button type="submit" id="delete_collections" name="action" value="delete" class="small"><i class="fa fa-minus-circle"></i> Delete selected</button>
-        <button type="submit" name="action" value="star" class="small"><i class="fa fa-minus-circle"></i> Star Selected</button>
-        <button type="submit" name="action" value="un-star" class="small"><i class="fa fa-minus-circle"></i> Un-star Selected</button>
+      <button type="submit" name="action" value="download-unread" class="small"><i class="fa fa-download"></i> Download Unread</button>
+      <button type="submit" name="action" value="download-all" class="small"><i class="fa fa-download"></i> Download All</button>
+        <button type="submit" name="action" value="star" class="small"><i class="fa fa-star"></i> Star</button>
+        <button type="submit" name="action" value="un-star" class="small"><i class="fa fa-star-half-full"></i> Un-star</button>
+        <button type="submit" id="delete_collections" name="action" value="delete" class="small-danger"><i class="fa fa-trash-o"></i> Delete</button>
       </p>
 
       {% if starred %}

securedrop/static/css/journalist.css

@@ -181,6 +181,12 @@ button.small, a.btn.small, .btn.small {
   padding: 3px 10px;
 }
 
+button.small-danger, a.btn.small-danger, .btn.small-danger {
+  background-color: #d55c5c;
+  font-size: small;
+  padding: 3px 10px;
+}
+
 .btn.primary {
   display: block;
   width: 50%;

securedrop/tests/test_unit_journalist.py

@@ -1,25 +1,25 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-import os
+
 from cStringIO import StringIO
-import unittest
-import zipfile
-import mock
-import time
 import datetime
-
-from flask_testing import TestCase
 from flask import url_for, escape
+from flask_testing import TestCase
+import mock
+import os
+import time
+import unittest
+import zipfile
 
 # Set environment variable so config.py uses a test environment
 os.environ['SECUREDROP_ENV'] = 'test'
-import config
 
+import common
+import config
 import crypto_util
 import journalist
-import common
-from db import (db_session, Source, Submission, Journalist, Reply,
-               InvalidPasswordLength)
+from db import (db_session, InvalidPasswordLength, Journalist, Reply, Source,
+                Submission)
 
 
 class TestJournalist(TestCase):
@@ -28,7 +28,7 @@ def create_app(self):
         return journalist.app
 
     def add_source_and_submissions(self):
-        sid = 'EQZGCJBRGISGOTC2NZVWG6LILJBHEV3CINNEWSCLLFTUWZJPKJFECLS2NZ4G4U3QOZCFKTTPNZMVIWDCJBBHMUDBGFHXCQ3R'
+        sid = crypto_util.hash_codename(crypto_util.genrandomid())
         codename = crypto_util.display_id()
         crypto_util.genkeypair(sid, codename)
         source = Source(sid, codename)
@@ -367,7 +367,8 @@ def test_admin_authorization_for_posts(self):
 
     def test_user_authorization_for_gets(self):
         urls = [url_for('index'), url_for('col', sid='1'),
-                url_for('doc', sid='1', fn='1'), url_for('edit_account')]
+                url_for('download_single_submission', sid='1', fn='1'),
+                url_for('edit_account')]
 
         for url in urls:
             res = self.client.get(url)
@@ -511,22 +512,115 @@ def test_delete_source_deletes_docs_on_disk(self):
         dir_source_docs = os.path.join(config.STORE_DIR, source.filesystem_id)
         self.assertFalse(os.path.exists(dir_source_docs))
 
-    def test_bulk_download(self):
-        source, files = self.add_source_and_submissions()
+    def test_download_selected_from_source(self):
+        sid = 'EQZGCJBRGISGOTC2NZVWG6LILJBHEV3CINNEWSCLLFTUWZJPKJFECLS2NZ4G4U3QOZCFKTTPNZMVIWDCJBBHMUDBGFHXCQ3R'
+        source = Source(sid, crypto_util.display_id())
+        db_session.add(source)
+        db_session.commit()
+        files = ['1-abc1-msg.gpg', '2-abc2-msg.gpg', '3-abc3-msg.gpg', '4-abc4-msg.gpg']
+        selected_files = files[:2]
+        unselected_files = files[2:]
+        common.setup_test_docs(sid, files)
 
         self._login_user()
-        rv = self.client.post('/bulk', data=dict(
-            action='download',
-            sid=source.filesystem_id,
-            doc_names_selected=files
-        ))
+        rv = self.client.post('/bulk',
+                              data=dict(action='download', sid=sid,
+                                        doc_names_selected=selected_files))
 
         self.assertEqual(rv.status_code, 200)
         self.assertEqual(rv.content_type, 'application/zip')
         self.assertTrue(zipfile.is_zipfile(StringIO(rv.data)))
-        self.assertTrue(zipfile.ZipFile(StringIO(rv.data)).getinfo(
-            os.path.join(source.journalist_filename, files[0])
-        ))
+
+        for file in selected_files:
+            self.assertTrue(
+                zipfile.ZipFile(StringIO(rv.data)).getinfo(
+                    os.path.join(source.journalist_filename, file))
+                )
+
+        for file in unselected_files:
+            try:
+                zipfile.ZipFile(StringIO(rv.data)).getinfo(
+                    os.path.join(source.journalist_filename, file))
+            except KeyError:
+                pass
+            else:
+                self.assertTrue(False)
+
+    def _setup_two_sources(self):
+        # Add two sources to the database
+        self.sid = crypto_util.hash_codename(crypto_util.genrandomid())
+        self.sid2 = crypto_util.hash_codename(crypto_util.genrandomid())
+        self.source = Source(self.sid, crypto_util.display_id())
+        self.source2 = Source(self.sid2, crypto_util.display_id())
+        db_session.add(self.source)
+        db_session.add(self.source2)
+        db_session.commit()
+
+        # The sources have made two submissions each, both being messages
+        self.files = ['1-s1-msg.gpg', '2-s1-msg.gpg']
+        common.setup_test_docs(self.sid, self.files)
+        self.files2 = ['1-s2-msg.gpg', '2-s2-msg.gpg']
+        common.setup_test_docs(self.sid2, self.files2)
+
+        # Mark the first message for each of the two sources read
+        for fn in (self.files[0], self.files2[0]):
+            s = Submission.query.filter(Submission.filename == fn).one()
+            s.downloaded = True
+        db_session.commit()
+
+
+
+    def test_download_unread_selected_sources(self):
+        self._setup_two_sources()
+        self._login_user()
+        resp = self.client.post('/col/process',
+                                data=dict(action='download-unread',
+                                          cols_selected=[self.sid, self.sid2]))
+
+        self.assertEqual(resp.status_code, 200)
+        self.assertEqual(resp.content_type, 'application/zip')
+        self.assertTrue(zipfile.is_zipfile(StringIO(resp.data)))
+
+        for file in (self.files[1], self.files2[1]):
+            self.assertTrue(
+                zipfile.ZipFile(StringIO(resp.data)).getinfo(
+                    os.path.join('unread', file))
+                )
+
+        for file in (self.files[0], self.files2[0]):
+            try:
+                zipfile.ZipFile(StringIO(resp.data)).getinfo(
+                    os.path.join('unread', file))
+            except KeyError:
+                pass
+            else:
+                self.assertTrue(False)
+
+    def test_download_all_selected_sources(self):
+        self._setup_two_sources()
+        self._login_user()
+        resp = self.client.post('/col/process',
+                                data=dict(action='download-all',
+                                          cols_selected=[self.sid2]))
+
+        self.assertEqual(resp.status_code, 200)
+        self.assertEqual(resp.content_type, 'application/zip')
+        self.assertTrue(zipfile.is_zipfile(StringIO(resp.data)))
+
+        for file in self.files2:
+            self.assertTrue(
+                zipfile.ZipFile(StringIO(resp.data)).getinfo(
+                    os.path.join('all', file))
+                )
+
+        for file in self.files:
+            try:
+                zipfile.ZipFile(StringIO(resp.data)).getinfo(
+                    os.path.join('all', file))
+            except KeyError:
+                pass
+            else:
+                self.assertTrue(False)
 
     def test_max_password_length(self):
         """Creating a Journalist with a password that is greater than the