Using pure python code to download and verify gpg encrypted files/messages

securedrop/requirements/test-requirements.in

@@ -7,6 +7,7 @@ py
 pytest
 pytest-cov
 pytest-mock
+requests[socks]
 selenium
 tbselenium
 pyvirtualdisplay

securedrop/requirements/test-requirements.txt

@@ -7,13 +7,16 @@
 attrs==17.4.0             # via pytest
 beautifulsoup4==4.6.0
 blinker==1.4
+certifi==2018.4.16        # via requests
+chardet==3.0.4            # via requests
 click==6.7                # via flask, pip-tools
 coverage==4.4.2           # via pytest-cov
 easyprocess==0.2.3        # via pyvirtualdisplay
 first==2.0.1              # via pip-tools
 flask-testing==0.7.1
 flask==0.12.2             # via flask-testing
 funcsigs==1.0.2           # via mock, pytest
+idna==2.7                 # via requests
 itsdangerous==0.24        # via flask
 jinja2==2.10              # via flask
 markupsafe==1.0           # via jinja2
@@ -22,11 +25,14 @@ pbr==3.1.1                # via mock
 pip-tools==1.11.0
 pluggy==0.6.0             # via pytest
 py==1.5.2
+pysocks==1.6.8            # via requests
 pytest-cov==2.5.1
 pytest-mock==1.7.1
 pytest==3.3.2
 pyvirtualdisplay==0.2.1
+requests[socks]==2.19.1
 selenium==3.13.0
 six==1.11.0               # via mock, pip-tools, pytest
 tbselenium==0.3.3
+urllib3==1.23             # via requests
 werkzeug==0.12.2          # via flask

securedrop/tests/functional/functional_test.py

@@ -202,15 +202,6 @@ def init_gpg(self):
             gpg.import_keys(open(keyfile).read())
         return gpg
 
-    def system(self, cmd):
-        """
-        Invoke a shell command. Primary replacement for os.system calls.
-        """
-        ret = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
-                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                               close_fds=True)
-        out, err = ret.communicate()
-        return out
 
     def setup(self, session_expiration=30):
 
@@ -222,6 +213,9 @@ def setup(self, session_expiration=30):
         instance_information_path = join(FUNCTIONAL_TEST_DIR,
                                          'instance_information.json')
 
+        env.create_directories()
+        self.gpg = env.init_gpg()
+
         if os.path.exists(instance_information_path):
             with open(instance_information_path) as fobj:
                 data = json.load(fobj)
@@ -244,8 +238,6 @@ def setup(self, session_expiration=30):
             self.mock_get_entropy_estimate = self.patcher2.start()
             self.mock_get_entropy_estimate.return_value = 8192
 
-            env.create_directories()
-            self.gpg = env.init_gpg()
             db.create_all()
 
             # Add our test user
@@ -255,9 +247,6 @@ def setup(self, session_expiration=30):
                                   password=valid_password,
                                   is_admin=True)
                 user.otp_secret = 'JHCOGO7VCER3EJ4L'
-                user.pw_salt = user._gen_salt()
-                user.pw_hash = user._scrypt_hash('WEjwn8ZyczDhQSK24YKM8C9a',
-                                                 user.pw_salt)
                 db.session.add(user)
                 db.session.commit()
             except IntegrityError:
@@ -279,7 +268,7 @@ def setup(self, session_expiration=30):
             # This user is required for our tests cases to login
             self.admin_user = {
                                 "name": "journalist",
-                                "password": "WEjwn8ZyczDhQSK24YKM8C9a",
+                                "password": "correct horse battery staple profanity oil chewy",
                                 "secret": "JHCOGO7VCER3EJ4L"}
             self.admin_user['totp'] = pyotp.TOTP(self.admin_user['secret'])
             self.sleep_time = 2

securedrop/tests/functional/journalist_navigation_steps.py

@@ -6,6 +6,7 @@
 import time
 import os
 import random
+import requests
 
 from os.path import abspath, realpath, dirname, join
 
@@ -51,19 +52,19 @@ def return_downloaded_content(self, url, cookies):
         :param cookies: the cookies to access
         :return: Content of the URL
         """
-        temp_cookie_file = tempfile.NamedTemporaryFile('w', delete=False)
-        data = {'url': url, 'cookies': cookies}
-        json.dump(data, temp_cookie_file)
-        temp_cookie_file.close()
-
-        cmd_path = abspath(join(dirname(realpath(__file__)),
-                                'download_content.py'))
-        # Now call the external program to handle the download
-        cmd = 'python {0} {1}'.format(cmd_path, temp_cookie_file.name)
-        if '.onion' in url:
-            cmd = 'nc -x 127.0.0.1:9150 ' + cmd
-        raw_content = self.system(cmd).strip()
-        return raw_content
+        proxies = None
+        if ".onion" in url:
+            proxies = {
+            'http': 'socks5h://127.0.0.1:9150',
+            'https': 'socks5h://127.0.0.1:9150'
+        }
+        r = requests.get(url, cookies=cookies, proxies=proxies,  stream=True)
+        if r.status_code != 200:
+            raise Exception("Failed to download the data.")
+        data = b""
+        for chunk in r.iter_content(1024):
+            data += chunk
+        return data
 
     def _input_text_in_login_form(self, username, password, token):
         self.driver.get(self.journalist_location + "/login")
@@ -640,14 +641,13 @@ def _journalist_downloads_message(self):
 
         # Downloading files with Selenium is tricky because it cannot automate
         # the browser's file download dialog. We can directly request the file
-        # using urllib2, but we need to pass the cookies for the logged in user
+        # using requests, but we need to pass the cookies for the logged in user
         # for Flask to allow this.
         def cookie_string_from_selenium_cookies(cookies):
-            cookie_strs = []
+            result = {}
             for cookie in cookies:
-                cookie_str = "=".join([cookie['name'], cookie['value']]) + ';'
-                cookie_strs.append(cookie_str)
-            return ' '.join(cookie_strs)
+                result[cookie['name']] = cookie['value']
+            return result
 
         cks = cookie_string_from_selenium_cookies(self.driver.get_cookies())
         raw_content = self.return_downloaded_content(file_url, cks)