Merge 0.3.9 into develop

The only merge conflicts were in the `securedrop/requirements/*.txt`
files, because:

1. Dependency version numbers were updated in 0.3.9, while
2. Dependencies were added/removed in develop.

Since the `securedrop/requirements/*.in` files merged successfully, we
resolved these merge conflicts simply by re-running `pip_update.sh`.

changelog.md

@@ -1,12 +1,26 @@
 # Changelog
 
+## 0.3.9
+
+Point release to fix some minor issues and update our Python dependencies.
+
+* Fix Unicode support regression and implement better Unicode tests (#1370)
+* Add OSSEC rule to ignore futile port scanning (#1374)
+* Update Apache AppArmor profile to allow access to webfonts and to execute uname (#1332, #1373)
+* Update Python dependencies of SD (#1379)
+* Fix a regression in the new install script (#1397)
+
+The issues for this release were tracked in the 0.3.9 milestone on Github: 
+https://github.com/freedomofpress/securedrop/milestones/0.3.9.
+
 ## 0.3.8
 
 * Re-include the pycrypto Python module to address the regression in 0.3.7 (#1344)
 * Switch to using bento boxes in Vagrantfile for more reproducible test environments
 * Minor fixes to update_version.sh
 
-The issues for this release were trackied with the 0.3.8 mileston on Github https://github.com/freedomofpress/securedrop/milestones/0.3.8
+The issues for this release were tracked in the 0.3.8 milestone on Github: 
+https://github.com/freedomofpress/securedrop/milestones/0.3.8
 
 ## 0.3.7
 

docs/conf.py

@@ -59,9 +59,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '0.3.8'
+version = '0.3.9'
 # The full version, including alpha/beta/rc tags.
-release = '0.3.8'
+release = '0.3.9'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/set_up_admin_tails.rst

@@ -81,8 +81,8 @@ key:
 .. code:: sh
 
     cd securedrop/
-    git checkout 0.3.8
-    git tag -v 0.3.8
+    git checkout 0.3.9
+    git tag -v 0.3.9
 
 You should see ``Good signature from "Freedom of the Press Foundation
 Master Signing Key"`` in the output of that last command.

install_files/ansible-base/group_vars/securedrop.yml

@@ -2,7 +2,7 @@
 # Variables that apply to both the app and monitor server go in this file
 # If the monitor or app server need different values define the variable in
 # hosts_vars/app.yml or host_vars/mon.yml host_vars/development.yml
-securedrop_app_code_version: "0.3.8"
+securedrop_app_code_version: "0.3.9"
 
 tor_wait_for_hidden_services: yes
 tor_hidden_services_parent_dir: "/var/lib/tor/services"

install_files/ansible-base/usr.sbin.apache2

@@ -11,6 +11,7 @@
 
   /bin/dash rix,
   /bin/touch rix,
+  /bin/uname rix,
   /dev/null w,
   /dev/urandom r,
   /etc/apache2/apache2.conf r,
@@ -186,6 +187,7 @@
   /var/www/securedrop/static/js/journalist.js r,
   /var/www/securedrop/static/js/libs/jquery-2.1.1.min.js r,
   /var/www/securedrop/static/js/source.js r,
+  /var/www/securedrop/static/fonts/fontawesome-webfont.eot r,
   /var/www/securedrop/static/fonts/fontawesome-webfont.ttf r,
   /var/www/securedrop/static/fonts/fontawesome-webfont.woff r,
   /var/www/securedrop/store.py r,

install_files/securedrop-app-code/DEBIAN/control

@@ -4,7 +4,7 @@ Priority: optional
 Maintainer: SecureDrop Team <[email protected]>
 Homepage: https://securedrop.org
 Package: securedrop-app-code
-Version: 0.3.8
+Version: 0.3.9
 Architecture: amd64
 Depends: python-pip,apparmor-utils,gnupg2,haveged,python,python-pip,secure-delete,sqlite,apache2-mpm-worker,libapache2-mod-wsgi,libapache2-mod-xsendfile,redis-server,supervisor
 Description: Packages the SecureDrop application code pip dependencies and apparmor profiles. This package will put the apparmor profiles in enforce mode. This package does use pip to install the pip wheelhouse

install_files/securedrop-app-code/DEBIAN/postinst

@@ -18,7 +18,8 @@ set -x
 
 case "$1" in
     configure)
-    pip install --no-index --find-links=/var/securedrop/wheelhouse -r /var/www/securedrop/requirements/securedrop-requirements.txt
+    pip install --no-index --find-links=/var/securedrop/wheelhouse --upgrade \
+      -r /var/www/securedrop/requirements/securedrop-requirements.txt
 
     chown -R www-data:www-data /var/www/securedrop
     chown www-data:www-data /var/www/document.wsgi

install_files/securedrop-app-code/usr/share/doc/securedrop-app-code/changelog.Debian

@@ -1,3 +1,15 @@
+securedrop-app-code (0.3.9) trusty; urgency=medium
+
+  * See changelog.md 
+
+ -- SecureDrop Team <[email protected]>  Tue, 13 Sep 2016 22:11:33 +0000
+
+securedrop-app-code (0.3.9-rc2) trusty; urgency=medium
+
+  * See changelog.md
+
+ -- SecureDrop Team <[email protected]>  Wed, 17 Aug 2016 00:03:15 +0000
+
 securedrop-app-code (0.3.8) trusty; urgency=medium
 
   * See changelog.md

install_files/securedrop-ossec-agent/DEBIAN/control

@@ -4,7 +4,7 @@ Priority: optional
 Maintainer: SecureDrop Team <[email protected]>
 Homepage: https://securedrop.org
 Package: securedrop-ossec-agent
-Version: 2.8.2+0.3.8
+Version: 2.8.2+0.3.9
 Architecture: amd64
 Depends: ossec-agent
 Replaces: ossec-agent

install_files/securedrop-ossec-server/DEBIAN/control

@@ -4,7 +4,7 @@ Priority: optional
 Maintainer: SecureDrop Team <[email protected]>
 Homepage: https://securedrop.org
 Package: securedrop-ossec-server
-Version: 2.8.2+0.3.8
+Version: 2.8.2+0.3.9
 Architecture: amd64
 Depends: ossec-server
 Replaces: ossec-server

install_files/securedrop-ossec-server/var/ossec/rules/local_rules.xml

@@ -115,3 +115,16 @@
     <options>no_email_alert</options>
   </rule>
 </group>
+
+<!--
+  Do not alert on attempted connections to the Tor HS on a port
+  the server is not listening on. Events are produced by
+  automated crawling/scanning.
+-->
+<group name="tor hs scans">
+  <rule id="200001" level="0">
+    <if_sid>1002</if_sid>
+    <match>connection_edge_process_relay_cell</match>
+    <options>no_email_alert</options>
+  </rule>
+</group>

pip_update.sh

@@ -28,15 +28,13 @@ trap "rm -rf ${venv}" EXIT
 virtualenv -p python2.7 $venv
 source "${venv}/bin/activate"
 
-# Install the most recent pip that pip-tools supports and the latest pip-tools
-# (must be done in order as the former is a dependency of the latter).
-pip install pip==8.1.1
+pip install --upgrade pip
 pip install pip-tools
 
 # Compile new requirements (.txt) files from our top-level dependency (.in)
 # files. See http://nvie.com/posts/better-package-management/
 for r in "securedrop" "test"; do
   # Maybe pip-tools will get its act together and standardize their cert-pinning
   # syntax and this line will break. One can only hope.
-  pip-compile -o "${r}-requirements.txt" "${r}-requirements.in"
+  pip-compile -U -o "${r}-requirements.txt" "${r}-requirements.in"
 done

securedrop/crypto_util.py

@@ -5,6 +5,7 @@
 
 from Crypto.Random import random
 import gnupg
+from gnupg._util import _is_stream, _make_binary_stream
 import scrypt
 
 import config
@@ -157,6 +158,9 @@ def encrypt(plaintext, fingerprints, output=None):
         fingerprints = [fingerprints, ]
     fingerprints = [fpr.replace(' ', '') for fpr in fingerprints]
 
+    if not _is_stream(plaintext):
+        plaintext = _make_binary_stream(plaintext, "utf_8")
+
     out = gpg.encrypt(plaintext,
                       *fingerprints,
                       output=output,
@@ -168,7 +172,7 @@ def encrypt(plaintext, fingerprints, output=None):
         raise CryptoException(out.stderr)
 
 
-def decrypt(secret, plain_text):
+def decrypt(secret, ciphertext):
     """
     >>> key = genkeypair('randomid', 'randomid')
     >>> decrypt('randomid', 'randomid',
@@ -177,8 +181,7 @@ def decrypt(secret, plain_text):
     'Goodbye, cruel world!'
     """
     hashed_codename = hash_codename(secret, salt=SCRYPT_GPG_PEPPER)
-    return gpg.decrypt(plain_text, passphrase=hashed_codename).data
-
+    return gpg.decrypt(ciphertext, passphrase=hashed_codename).data
 
 if __name__ == "__main__":
     import doctest

securedrop/requirements/securedrop-requirements.txt

@@ -4,27 +4,25 @@
 #
 #    pip-compile --output-file securedrop-requirements.txt securedrop-requirements.in
 #
-click==6.6                # via rq
-colorama==0.3.7           # via qrcode
+click==6.6                # via flask, rq
 cssmin==0.2.0
-Flask-Assets==0.11
+Flask-Assets==0.12
 Flask-WTF==0.12
-Flask==0.10.1             # via flask-assets, flask-wtf
-future==0.15.2            # via pyotp
+Flask==0.11.1             # via flask-assets, flask-wtf
 gnupg==2.0.2
 itsdangerous==0.24        # via flask
 Jinja2==2.8               # via flask
 jsmin==2.2.1
 MarkupSafe==0.23          # via jinja2
-psutil==4.1.0
+psutil==4.3.1
 pycrypto==2.6.1
-pyotp==2.1.1
-qrcode==5.2.2
+pyotp==2.2.1
+qrcode==5.3
 redis==2.10.5
 rq==0.6.0
-scrypt==0.7.1
+scrypt==0.8.0
 six==1.10.0               # via qrcode
-SQLAlchemy==1.0.13
-webassets==0.11.1         # via flask-assets
-Werkzeug==0.11.9          # via flask, flask-wtf
+SQLAlchemy==1.0.15
+webassets==0.12.0         # via flask-assets
+Werkzeug==0.11.11         # via flask, flask-wtf
 WTForms==2.1              # via flask-wtf

securedrop/requirements/test-requirements.txt

@@ -4,22 +4,22 @@
 #
 #    pip-compile --output-file test-requirements.txt test-requirements.in
 #
-beautifulsoup4==4.4.1
-click==6.6                # via pip-tools
+beautifulsoup4==4.5.1
+click==6.6                # via flask, pip-tools
 coverage==4.2             # via pytest-cov
 first==2.0.1              # via pip-tools
-Flask-Testing==0.4.2
-Flask==0.10.1             # via flask-testing
+Flask-Testing==0.6.1
+Flask==0.11.1             # via flask-testing
 funcsigs==1.0.2           # via mock
 itsdangerous==0.24        # via flask
 Jinja2==2.8               # via flask
 MarkupSafe==0.23          # via jinja2
 mock==2.0.0
-pbr==1.9.1                # via mock
-pip-tools==1.6.5
+pbr==1.10.0               # via mock
+pip-tools==1.7.0
 py==1.4.31
 pytest-cov==2.3.1
-pytest==2.9.1
+pytest==3.0.2
 selenium==2.53.6
 six==1.10.0               # via mock, pip-tools
-Werkzeug==0.11.9          # via flask
+Werkzeug==0.11.11         # via flask

securedrop/tests/test_unit_integration.py

@@ -258,8 +258,21 @@ def test_submit_file(self):
     def test_reply_normal(self):
         self.helper_test_reply("This is a test reply.", True)
 
-    def test_reply_unicode(self):
-        self.helper_test_reply("Teşekkürler", True)
+    def test_unicode_reply_with_ansi_env(self):
+        # This makes python-gnupg handle encoding equivalent to if we were
+        # running SD in an environment where os.getenv("LANG") == "C".
+        # Unfortunately, with the way our test suite is set up simply setting
+        # that env var here will not have the desired effect. Instead we
+        # monkey-patch the GPG object that is called crypto_util to imitate the
+        # _encoding attribute it would have had it been initialized in a "C"
+        # environment. See
+        # https://github.com/freedomofpress/securedrop/issues/1360 for context.
+        old_encoding = crypto_util.gpg._encoding
+        crypto_util.gpg._encoding = "ansi_x3.4_1968"
+        try:
+            self.helper_test_reply("ᚠᛇᚻ᛫ᛒᛦᚦ᛫ᚠᚱᚩᚠᚢᚱ᛫ᚠᛁᚱᚪ᛫ᚷᛖᚻᚹᛦᛚᚳᚢᛗ", True)
+        finally:
+            crypto_util.gpg._encoding = old_encoding
 
     def _can_decrypt_with_key(self, msg, key_fpr, passphrase=None):
         """

securedrop/version.py

@@ -1 +1 @@
-__version__ = '0.3.8'
+__version__ = '0.3.9'

tails_files/install.sh

@@ -219,7 +219,7 @@ function lookup_document_aths_url()
   # Last shot before prompting: check for an existing desktop icon specifically
   # for the Document Interface. If found, we can extract the URL from there.
   elif [ -e "${amnesia_desktop}/document.desktop" ] ; then
-    app_document_aths="$(grep ^Exec=/usr/local/bin/tor-browser | awk '{ print $2 }')"
+    app_document_aths="$(grep ^Exec=/usr/local/bin/tor-browser "${amnesia_desktop}/document.desktop" | awk '{ print $2 }')"
   # Couldn't find it anywhere. We'll have to prompt!
   else
     echo "Could not find Document Interface ATHS info, prompting interactively..." 1>&2
@@ -240,7 +240,7 @@ function lookup_source_ths_url()
     app_source_ths="$source_ths_url_global"
   # Failing that, check for the public THS URL in an existing Desktop icon.
   elif grep -q -P '^Exec=/usr/local/bin/tor-browser\s+[a-z2-7]{16}\.onion' "${amnesia_desktop}/source.desktop" ; then
-    app_source_ths="$(grep ^Exec=/usr/local/bin/tor-browser | awk '{ print $2 }')"
+    app_source_ths="$(grep ^Exec=/usr/local/bin/tor-browser "${amnesia_desktop}/source.desktop" | awk '{ print $2 }')"
   # Couldn't find it anywhere. We'll have to prompt!
   else
     echo "Could not find Source Interface Onion URL, prompting interactively..." 1>&2