Add pip, setuptools and supervisor to develop requirements

These requirements were installed via pip in the Dev/Admin Docker containers. We should instead add them to the develop-requirements file and pin hashes.

The `--allow-unsafe` flags should not impact security: it is to allow pip to pin setuptools and pip-tools (see issue #4686).

Makefile

@@ -29,6 +29,7 @@ update-admin-pip-requirements:  ## Update admin requirements.
 update-python3-requirements:  ## Update Python 3 requirements with pip-compile.
 	@echo "███ Updating Python 3 requirements files..."
 	@$(DEVSHELL) pip-compile --generate-hashes \
+		--allow-unsafe \
 		--output-file requirements/python3/develop-requirements.txt \
 		../admin/requirements-ansible.in \
 		../admin/requirements.in \

admin/Dockerfile

@@ -17,7 +17,5 @@ ENV VIRTUAL_ENV /opt/.venv
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 COPY requirements-dev.txt .
 RUN pip3 install --require-hashes -r requirements-dev.txt
-# Now also pin pip due to https://github.com/jazzband/pip-tools/issues/853
-RUN pip3 install pip==19.1
 
 RUN chown -R $USER_NAME /opt

securedrop/dockerfiles/xenial/python3/Dockerfile

@@ -49,11 +49,8 @@ RUN wget https://github.com/mozilla/geckodriver/releases/download/v0.24.0/geckod
 
 COPY requirements requirements
 RUN python3 -m venv /opt/venvs/securedrop-app-code && \
-    /opt/venvs/securedrop-app-code/bin/pip3 install pip-tools && \
     /opt/venvs/securedrop-app-code/bin/pip3 install --require-hashes -r requirements/python3/securedrop-app-code-requirements.txt && \
-    /opt/venvs/securedrop-app-code/bin/pip3 install -r requirements/python3/test-requirements.txt && \
-    /opt/venvs/securedrop-app-code/bin/pip3 install supervisor && \
-    /opt/venvs/securedrop-app-code/bin/pip3 install --upgrade setuptools  # Fixes #4036 pybabel requires latest version of setuptools
+    /opt/venvs/securedrop-app-code/bin/pip3 install --require-hashes -r requirements/python3/test-requirements.txt
 
 RUN if test $USER_NAME != root ; then useradd --no-create-home --home-dir /tmp --uid $USER_ID $USER_NAME && echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers ; fi && \
     cp -r /root/.local /tmp/ && chmod +x /tmp/.local/tbb/tor-browser_en-US/Browser/firefox && chmod -R 777 /tmp/.local && \

securedrop/requirements/python3/develop-requirements.in

@@ -12,15 +12,19 @@ mypy
 # Needed for ansible network filter
 # http://docs.ansible.com/ansible/latest/playbooks_filters_ipaddr.html
 netaddr
+# Now also pin pip due to https://github.com/jazzband/pip-tools/issues/853
+pip==19.1
 pip-tools>=4.0.0
 pyenchant
 pylint
 pytest-xdist
 python-vagrant
 safety>=1.8.4
+setuptools
 sphinx
 sphinx-autobuild
 sphinx_rtd_theme
+supervisor
 testinfra
 urllib3>=1.25.3
 yamllint

securedrop/requirements/python3/develop-requirements.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --generate-hashes --output-file=requirements/python3/develop-requirements.txt ../admin/requirements-ansible.in ../admin/requirements.in requirements/python3/develop-requirements.in
+#    pip-compile --allow-unsafe --generate-hashes --output-file=requirements/python3/develop-requirements.txt ../admin/requirements-ansible.in ../admin/requirements.in requirements/python3/develop-requirements.in
 #
 alabaster==0.7.10 \
     --hash=sha256:2eef172f44e8d301d25aff8068fddd65f767a3f04b5f15b0f4922f113aa1c732 \
@@ -317,6 +317,10 @@ mccabe==0.6.1 \
     --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \
     --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f \
     # via flake8, pylint
+meld3==2.0.0 \
+    --hash=sha256:1efda676264490db2e30bfb81b27a918cc6d9c2de6d609491aa43410b9537eb9 \
+    --hash=sha256:a78cdcb6c0342b7ac82d900b584bd918ef72f86b60cdbf04754f840c070eb20c \
+    # via supervisor
 molecule==2.22 \
     --hash=sha256:12fa4231ed69c6e7f50432588eaace36cea917a8c73c1751269ce55df32ced24 \
     --hash=sha256:d9d7621167041ae2a8eb19f1f8dc23c071cdab2cd3ca80655e2c8796b4c00e09
@@ -386,7 +390,7 @@ pre-commit==1.18.3 \
     --hash=sha256:1d3c0587bda7c4e537a46c27f2c84aa006acc18facf9970bf947df596ce91f3f \
     --hash=sha256:fa78ff96e8e9ac94c748388597693f18b041a181c94a4f039ad20f45287ba44a \
     # via molecule
-prompt-toolkit==2.0.9 \
+prompt_toolkit==2.0.9 \
     --hash=sha256:11adf3389a996a6d45cc277580d0d53e8a5afd281d0c9ec71b28e6f121463780 \
     --hash=sha256:2519ad1d8038fd5fc8e770362237ad0364d16a7650fb5724af6997ed5515e3c1 \
     --hash=sha256:977c6583ae813a37dc1c2e1b715892461fcbdaa57f6fc62f33a528c4886c8f55
@@ -566,6 +570,9 @@ stevedore==1.28.0 \
     --hash=sha256:e3d96b2c4e882ec0c1ff95eaebf7b575a779fd0ccb4c741b9832bed410d58b3d \
     --hash=sha256:f1c7518e7b160336040fee272174f1f7b29a46febb3632502a8f2055f973d60b \
     # via bandit
+supervisor==4.0.4 \
+    --hash=sha256:43e87c7b572a94acdb586aaebb06844dae1aa02856b984c5a738032abd753fb7 \
+    --hash=sha256:9644990d21a1ba03b1a7ac5e9a0c0c62e12822e258f9e98f4a0b128461b3f10a
 tabulate==0.8.3 \
     --hash=sha256:8af07a39377cee1103a5c8b3330a421c2d99b9141e9cc5ddd2e3263fea416943 \
     # via molecule
@@ -644,7 +651,10 @@ zipp==0.6.0 \
     --hash=sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335 \
     # via importlib-metadata
 
-# WARNING: The following packages were not pinned, but pip requires them to be
-# pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag.
-# pip==19.2.3               # via safety
-# setuptools==41.2.0        # via ansible, pytest, sphinx
+# The following packages are considered to be unsafe in a requirements file:
+pip==19.1 \
+    --hash=sha256:8f59b6cf84584d7962d79fd1be7a8ec0eb198aa52ea864896551736b3614eee9 \
+    --hash=sha256:d9137cb543d8a4d73140a3282f6d777b2e786bb6abb8add3ac5b6539c82cd624
+setuptools==41.4.0 \
+    --hash=sha256:7eae782ccf36b790c21bde7d86a4f303a441cd77036b25c559a602cf5186ce4d \
+    --hash=sha256:8d01f7ee4191d9fdcd9cc5796f75199deccb25b154eba82d44d6a042cf873670