fix: Unify python dependency installation and update to vulnerability free versions

Dockerfile

@@ -5,7 +5,6 @@ ENV LANG=en_US.UTF-8 \
     WORKER_DATA_DIR='/var/lib/f8a_worker/worker_data' \
     # home directory
     HOME='/workdir' \
-    F8A_UTILS_VERSION=3bca34e \
     # place for alembic migrations
     ALEMBIC_DIR='/alembic'
 
@@ -21,8 +20,6 @@ COPY requirements.txt /tmp/f8a_worker/
 RUN cd /tmp/f8a_worker/ && \
     pip3 install -r requirements.txt
 
-RUN pip3 install git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@${F8A_UTILS_VERSION}
-RUN pip3 install git+https://[email protected]/fabric8-analytics/fabric8-analytics-version-comparator.git#egg=f8a_version_comparator
 COPY alembic.ini hack/run-db-migrations.sh ${ALEMBIC_DIR}/
 COPY alembic/ ${ALEMBIC_DIR}/alembic
 COPY selinon-2.0.0.tar.gz ${HOME}

Dockerfile.rhel

@@ -5,7 +5,6 @@ ENV LANG=en_US.UTF-8 \
     WORKER_DATA_DIR='/var/lib/f8a_worker/worker_data' \
     # home directory
     HOME='/workdir' \
-    F8A_UTILS_VERSION=3bca34e \
     # place for alembic migrations
     ALEMBIC_DIR='/alembic'
 
@@ -21,8 +20,6 @@ COPY requirements.txt /tmp/f8a_worker/
 RUN cd /tmp/f8a_worker/ && \
     pip3 install -r requirements.txt
 
-RUN pip3 install git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@${F8A_UTILS_VERSION}
-RUN pip3 install git+https://[email protected]/fabric8-analytics/fabric8-analytics-version-comparator.git#egg=f8a_version_comparator
 COPY alembic.ini hack/run-db-migrations.sh ${ALEMBIC_DIR}/
 COPY alembic/ ${ALEMBIC_DIR}/alembic
 COPY selinon-2.0.0.tar.gz ${HOME}

requirements.in

@@ -1,26 +1,60 @@
-# normally pulled in by kombu, but version 2.2.0 is broken
-amqp<=2.1.4
-# also handles Celery as a requirement
-selinon[celery]
-sqlalchemy
-psycopg2
-lxml
-beautifulsoup4
-# We install a patched version from dnf as we cannot use requests from PyPI - we need own certificates
-#requests
+amqp
+anymarkup-core
 anymarkup
+attrs
+beautifulsoup4
+billiard
+boto3
+boto
+botocore
+certifi
+chardet
+click
+codegen
+colorama
+configobj
+docutils
+flake8-polyfill
+flake8
+git2json
+gitdb
+gitpython<=3.1.0
+graphviz
+idna
+importlib-metadata
+jmespath
 jsl
+json5
 jsonschema
-unidiff
-requests
-requests-futures
-git2json
-gitpython
-# Amazon AWS SQS
-# Celery transparently uses boto
-boto
-boto3
-semantic_version
-radon==3.0.1
-watchdog
+kombu
+logutils
+lxml
+mando
+mccabe
+psycopg2-binary
+pycodestyle
+pyflakes
+pyrsistent
+python-dateutil
+pytz
+pyyaml
+radon<=3.0.1
+rainbow-logging-handler
 raven
+requests-futures
+requests
+s3transfer
+selinon[celery]==1.0.0
+semantic-version
+six
+smmap
+soupsieve
+sqlalchemy
+toml<=0.9.4
+urllib3
+vine
+werkzeug
+xmltodict
+zipp
+git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@21ea911
+git+https://github.com/fabric8-analytics/fabric8-analytics-version-comparator.git@8a57ac7

requirements.txt

@@ -2,60 +2,69 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --output-file requirements.txt requirements.in
+#    pip-compile
 #
-amqp==2.1.4
-anymarkup-core==0.7.1     # via anymarkup
-anymarkup==0.7.0
-argh==0.26.2              # via watchdog
-beautifulsoup4==4.6.3
-billiard==3.5.0.4         # via celery
-boto3==1.9.44
-boto==2.49.0
-botocore==1.12.44         # via boto3, s3transfer
-celery==4.2.1             # via selinon
-certifi==2018.10.15       # via requests
-chardet==3.0.4            # via requests
-click==7.0                # via selinon
-codegen==1.0              # via selinon
-colorama==0.3.9           # via radon, rainbow-logging-handler
-configobj==5.0.6          # via anymarkup
-docutils==0.14            # via botocore
-flake8-polyfill==1.0.2    # via radon
-flake8==3.6.0             # via flake8-polyfill
-git2json==0.2.3
-graphviz==0.10.1          # via selinon
-gitpython==3.1.0
-idna==2.7                 # via requests
-jmespath==0.9.3           # via boto3, botocore
-jsl==0.2.4
-json5==0.6.1              # via anymarkup
-jsonschema==2.6.0
-kombu==4.2.1              # via celery
-logutils==0.3.5           # via rainbow-logging-handler
-lxml==4.2.5
-mando==0.6.4              # via radon
-mccabe==0.6.1             # via flake8
-pathtools==0.1.2          # via watchdog
-psycopg2==2.7.6.1
-pycodestyle==2.4.0        # via flake8
-pyflakes==2.0.0           # via flake8
-python-dateutil==2.7.5    # via botocore
-pytz==2018.7              # via celery
-pyyaml==3.13              # via anymarkup, selinon, watchdog
-radon==3.0.1
-rainbow-logging-handler==2.2.2  # via selinon
-raven==6.9.0
-requests==2.20.1
-requests-futures==0.9.7
-s3transfer==0.1.13        # via boto3
-semantic-version==2.6.0
-six==1.11.0               # via anymarkup-core, configobj, mando, python-dateutil
-sqlalchemy==1.2.14
-toml==0.9.4               # via anymarkup
-unidiff==0.5.5
-urllib3==1.24.1           # via botocore, requests
-vine==1.1.4               # via amqp
-watchdog==0.9.0
-werkzeug==0.14.1          # via flask
-xmltodict==0.11.0         # via anymarkup
+amqp==2.6.1               # via -r requirements.in, kombu
+anymarkup-core==0.8.1     # via -r requirements.in, anymarkup
+anymarkup==0.8.1          # via -r requirements.in
+attrs==19.3.0             # via -r requirements.in, jsonschema
+beautifulsoup4==4.9.1     # via -r requirements.in
+billiard==3.6.3.0         # via -r requirements.in, celery
+boto3==1.14.34            # via -r requirements.in
+boto==2.49.0              # via -r requirements.in
+botocore==1.17.34         # via -r requirements.in, boto3, s3transfer
+celery==4.4.7             # via selinon
+certifi==2020.6.20        # via -r requirements.in, requests
+chardet==3.0.4            # via -r requirements.in, requests
+click==7.1.2              # via -r requirements.in, anymarkup, selinon
+codegen==1.0              # via -r requirements.in, selinon
+colorama==0.4.3           # via -r requirements.in, radon, rainbow-logging-handler
+configobj==5.0.6          # via -r requirements.in, anymarkup
+docutils==0.15.2          # via -r requirements.in, botocore
+git+https://github.com/fabric8-analytics/fabric8-analytics-version-comparator.git@8a57ac7  # via -r requirements.in
+git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@21ea911  # via -r requirements.in
+flake8-polyfill==1.0.2    # via -r requirements.in, radon
+flake8==3.8.3             # via -r requirements.in, flake8-polyfill
+git2json==0.2.3           # via -r requirements.in
+gitdb==4.0.5              # via -r requirements.in, gitpython
+gitpython==3.1.0          # via -r requirements.in
+graphviz==0.14.1          # via -r requirements.in, selinon
+idna==2.10                # via -r requirements.in, requests
+importlib-metadata==1.7.0  # via -r requirements.in, flake8, jsonschema, kombu
+jmespath==0.10.0          # via -r requirements.in, boto3, botocore
+jsl==0.2.4                # via -r requirements.in
+json5==0.9.5              # via -r requirements.in, anymarkup
+jsonschema==3.2.0         # via -r requirements.in, selinon
+kombu==4.6.11             # via -r requirements.in, celery
+logutils==0.3.5           # via -r requirements.in, rainbow-logging-handler
+lxml==4.5.2               # via -r requirements.in, fabric8-analytics-utils
+mando==0.6.4              # via -r requirements.in, radon
+mccabe==0.6.1             # via -r requirements.in, flake8
+psycopg2-binary==2.8.5    # via -r requirements.in
+pycodestyle==2.6.0        # via -r requirements.in, flake8
+pyflakes==2.2.0           # via -r requirements.in, flake8
+pyrsistent==0.16.0        # via -r requirements.in, jsonschema
+python-dateutil==2.8.1    # via -r requirements.in, botocore
+pytz==2020.1              # via -r requirements.in, celery
+pyyaml==5.3.1             # via -r requirements.in, anymarkup, selinon
+radon==3.0.1              # via -r requirements.in
+rainbow-logging-handler==2.2.2  # via -r requirements.in, selinon
+raven==6.10.0             # via -r requirements.in
+requests-futures==1.0.0   # via -r requirements.in
+requests==2.24.0          # via -r requirements.in, fabric8-analytics-utils, requests-futures
+s3transfer==0.3.3         # via -r requirements.in, boto3
+selinon[celery]==1.0.0    # via -r requirements.in
+semantic-version==2.8.5   # via -r requirements.in
+six==1.15.0               # via -r requirements.in, anymarkup-core, configobj, jsonschema, mando, pyrsistent, python-dateutil
+smmap==3.0.4              # via -r requirements.in, gitdb
+soupsieve==2.0.1          # via -r requirements.in, beautifulsoup4
+sqlalchemy==1.3.18        # via -r requirements.in
+toml==0.9.4               # via -r requirements.in, anymarkup
+urllib3==1.25.10          # via -r requirements.in, botocore, requests
+vine==1.3.0               # via -r requirements.in, amqp, celery
+werkzeug==1.0.1           # via -r requirements.in
+xmltodict==0.12.0         # via -r requirements.in, anymarkup
+zipp==3.1.0               # via -r requirements.in, importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

setup.py

@@ -8,10 +8,19 @@
 
 def get_requirements():
     """Parse all packages mentioned in the 'requirements.txt' file."""
-    with open('requirements.txt') as fd:
-        return fd.read().splitlines()
+    with open('requirements.in') as fd:
+        lines = fd.read().splitlines()
+        reqs, dep_links = [], []
+        for line in lines:
+            if line.startswith('git+'):
+                dep_links.append(line)
+            else:
+                reqs.append(line)
+        return reqs, dep_links
 
 
+reqs, dep_links = get_requirements()
+
 setup(
     name='f8a_worker',
     version='0.2',
@@ -30,7 +39,8 @@ def get_requirements():
     },
     packages=find_packages(exclude=['tests', 'tests.*']),
     include_package_data=True,
-    install_requires=get_requirements(),
+    install_requires=reqs,
+    dependency_links=dep_links,
     author='Pavel Odvody',
     author_email='[email protected]',
     description='fabric8-analytics workers & utilities',

tests/requirements.in

@@ -1,10 +1,29 @@
-datadiff
+astroid
+atomicwrites
+attrs
+certifi
+chardet
+codecov
+coverage
 flexmock
+idna
+importlib-metadata
+isort
+lazy-object-proxy
+mccabe
+more-itertools
+pluggy
+py
 pylint
-pytest==3.*
-pytest-timeout
-pytest-rerunfailures
-pytest-cov
-pytest-mock
-codecov
+pytest-cov<=2.6.0
+pytest-mock<=1.10.0
+pytest-rerunfailures<=5.0
+pytest-timeout<=1.3.3
+pytest<=3.10.1requests
 requests
+six
+toml<=0.9.4
+typed-ast
+urllib3
+wrapt
+zipp

tests/requirements.txt

@@ -2,31 +2,37 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --output-file requirements.txt requirements.in
+#    pip-compile
 #
-astroid==2.1.0            # via pylint
-atomicwrites==1.2.1       # via pytest
-attrs==18.2.0             # via pytest
-certifi==2018.10.15       # via requests
-chardet==3.0.4            # via requests
-codecov==2.0.15
-coverage==4.5.2           # via codecov, pytest-cov
-datadiff==2.0.0
-flexmock==0.10.2
-idna==2.7                 # via requests
-isort==4.3.4              # via pylint
-lazy-object-proxy==1.3.1  # via astroid
-mccabe==0.6.1             # via pylint
-more-itertools==4.3.0     # via pytest
-pluggy==0.8.0             # via pytest
-py==1.7.0                 # via pytest
-pylint==2.2.2
-pytest-cov==2.6.0
-pytest-mock==1.10.0
-pytest-rerunfailures==5.0
-pytest-timeout==1.3.3
-pytest==3.10.1
-requests==2.20.1
-six==1.11.0               # via astroid, more-itertools, pytest
-urllib3==1.24.1           # via requests
-wrapt==1.10.11            # via astroid
+astroid==2.4.2            # via -r requirements.in, pylint
+atomicwrites==1.4.0       # via -r requirements.in, pytest
+attrs==19.3.0             # via -r requirements.in, pytest
+certifi==2020.6.20        # via -r requirements.in, requests
+chardet==3.0.4            # via -r requirements.in, requests
+codecov==2.1.8            # via -r requirements.in
+coverage==5.2.1           # via -r requirements.in, codecov, pytest-cov
+flexmock==0.10.4          # via -r requirements.in
+idna==2.10                # via -r requirements.in, requests
+importlib-metadata==1.7.0  # via pluggy
+isort==4.3.21             # via -r requirements.in, pylint
+lazy-object-proxy==1.4.3  # via -r requirements.in, astroid
+mccabe==0.6.1             # via -r requirements.in, pylint
+more-itertools==8.4.0     # via -r requirements.in, pytest
+pluggy==0.13.1            # via -r requirements.in, pytest
+py==1.9.0                 # via -r requirements.in, pytest
+pylint==2.5.3             # via -r requirements.in
+pytest-cov==2.6.0         # via -r requirements.in
+pytest-mock==1.10.0       # via -r requirements.in
+pytest-rerunfailures==5.0  # via -r requirements.in
+pytest-timeout==1.3.3     # via -r requirements.in
+pytest==3.10.1            # via -r requirements.in, pytest-cov, pytest-mock, pytest-rerunfailures, pytest-timeout
+requests==2.24.0          # via codecov
+six==1.15.0               # via -r requirements.in, astroid, pytest
+toml==0.9.4               # via -r requirements.in, pylint
+typed-ast==1.4.1          # via -r requirements.in, astroid
+urllib3==1.25.10          # via -r requirements.in, requests
+wrapt==1.12.1             # via -r requirements.in, astroid
+zipp==3.1.0               # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools