From 32bf84cd7878d03529cc500a952304d92a5cfc7c Mon Sep 17 00:00:00 2001 From: jparsai Date: Thu, 20 Aug 2020 21:45:06 +0530 Subject: [PATCH] fix: Unify python dependency installation and update to vulnerability free versions (#912) * Resolve vulnerabilities, unified dependency handling. * Unify python dependency installation and update to vulnerability free versions * Unify python dependency installation and update to vulnerability free versions * [APPAI-1432] Unify python dependency installation and update to vulnerability free versions. * [APPAI-1432] Adding radon to fix PR build. * [APPAI-1432] Removed indirect dependencies from requiremnts.in * [APPAI-1432] Removed indirect dependencies from requiremnts.in * Removed usage of deprecated dependency_links from setup.py --- Dockerfile | 3 -- Dockerfile.rhel | 3 -- requirements.in | 38 +++++++-------- requirements.txt | 105 ++++++++++++++++++++--------------------- setup.py | 15 ++++-- tests/requirements.in | 14 ++---- tests/requirements.txt | 57 ++++++++++++---------- 7 files changed, 115 insertions(+), 120 deletions(-) diff --git a/Dockerfile b/Dockerfile index b259f7819..51c69801f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,6 @@ ENV LANG=en_US.UTF-8 \ WORKER_DATA_DIR='/var/lib/f8a_worker/worker_data' \ # home directory HOME='/workdir' \ - F8A_UTILS_VERSION=3bca34e \ # place for alembic migrations ALEMBIC_DIR='/alembic' @@ -21,8 +20,6 @@ COPY requirements.txt /tmp/f8a_worker/ RUN cd /tmp/f8a_worker/ && \ pip3 install -r requirements.txt -RUN pip3 install git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@${F8A_UTILS_VERSION} -RUN pip3 install git+https://git@github.com/fabric8-analytics/fabric8-analytics-version-comparator.git#egg=f8a_version_comparator COPY alembic.ini hack/run-db-migrations.sh ${ALEMBIC_DIR}/ COPY alembic/ ${ALEMBIC_DIR}/alembic diff --git a/Dockerfile.rhel b/Dockerfile.rhel index 8ca1e03d6..eeb9249c7 100644 --- a/Dockerfile.rhel +++ b/Dockerfile.rhel @@ -5,7 +5,6 @@ ENV LANG=en_US.UTF-8 \ WORKER_DATA_DIR='/var/lib/f8a_worker/worker_data' \ # home directory HOME='/workdir' \ - F8A_UTILS_VERSION=3bca34e \ # place for alembic migrations ALEMBIC_DIR='/alembic' @@ -21,8 +20,6 @@ COPY requirements.txt /tmp/f8a_worker/ RUN cd /tmp/f8a_worker/ && \ pip3 install -r requirements.txt -RUN pip3 install git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@${F8A_UTILS_VERSION} -RUN pip3 install git+https://git@github.com/fabric8-analytics/fabric8-analytics-version-comparator.git#egg=f8a_version_comparator COPY alembic.ini hack/run-db-migrations.sh ${ALEMBIC_DIR}/ COPY alembic/ ${ALEMBIC_DIR}/alembic diff --git a/requirements.in b/requirements.in index 88865b666..9aa8e28eb 100644 --- a/requirements.in +++ b/requirements.in @@ -1,26 +1,20 @@ -# normally pulled in by kombu, but version 2.2.0 is broken -amqp<=2.1.4 -# also handles Celery as a requirement -selinon[celery] -sqlalchemy -psycopg2 -lxml -beautifulsoup4 -# We install a patched version from dnf as we cannot use requests from PyPI - we need own certificates -#requests -anymarkup -jsl -jsonschema -unidiff requests -requests-futures +anymarkup +beautifulsoup4 +boto3 +botocore git2json gitpython -# Amazon AWS SQS -# Celery transparently uses boto -boto -boto3 -semantic_version -radon==3.0.1 -watchdog +jsl +jsonschema +lxml +pyyaml raven +requests-futures +selinon[celery]==1.0.0 +semantic-version +sqlalchemy +tenacity +toml<=0.9.4 +werkzeug +f8a_utils @ git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@44c123b#egg=f8a_utils \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index b251f7351..5fd9bb868 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,62 +2,61 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file requirements.txt requirements.in +# pip-compile # -amqp==2.1.4 -anymarkup-core==0.7.1 # via anymarkup -anymarkup==0.7.0 -argh==0.26.2 # via watchdog -beautifulsoup4==4.6.3 -billiard==3.5.0.4 # via celery -boto3==1.9.44 -boto==2.49.0 -botocore==1.12.44 # via boto3, s3transfer -celery==4.2.1 # via selinon -certifi==2018.10.15 # via requests +amqp==2.6.1 # via kombu +anymarkup-core==0.8.1 # via anymarkup +anymarkup==0.8.1 # via -r requirements.in +attrs==19.3.0 # via jsonschema +beautifulsoup4==4.9.1 # via -r requirements.in +billiard==3.6.3.0 # via celery +boto3==1.14.43 # via -r requirements.in +botocore==1.17.43 # via -r requirements.in, boto3, s3transfer +celery==4.4.7 # via selinon +certifi==2020.6.20 # via requests chardet==3.0.4 # via requests -click==7.0 # via selinon +click==7.1.2 # via anymarkup, selinon codegen==1.0 # via selinon -colorama==0.3.9 # via radon, rainbow-logging-handler +colorama==0.4.3 # via rainbow-logging-handler configobj==5.0.6 # via anymarkup -docutils==0.14 # via botocore -flake8-polyfill==1.0.2 # via radon -flake8==3.6.0 # via flake8-polyfill -git2json==0.2.3 -graphviz==0.10.1 # via selinon -gitpython==3.1.0 -idna==2.7 # via requests -jmespath==0.9.3 # via boto3, botocore -jsl==0.2.4 -json5==0.6.1 # via anymarkup -jsonschema==2.6.0 -kombu==4.2.1 # via celery +docutils==0.15.2 # via botocore +git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@44c123b#egg=f8a_utils # via -r requirements.in +git+https://github.com/fabric8-analytics/fabric8-analytics-version-comparator.git@8a57ac7#egg=f8a_version_comparator # via f8a-utils +git2json==0.2.3 # via -r requirements.in +gitdb==4.0.5 # via gitpython +gitpython==3.1.7 # via -r requirements.in +graphviz==0.14.1 # via selinon +idna==2.10 # via requests +importlib-metadata==1.7.0 # via jsonschema, kombu +jmespath==0.10.0 # via boto3, botocore +jsl==0.2.4 # via -r requirements.in +json5==0.9.5 # via anymarkup +jsonschema==3.2.0 # via -r requirements.in, selinon +kombu==4.6.11 # via celery logutils==0.3.5 # via rainbow-logging-handler -lxml==4.2.5 -mando==0.6.4 # via radon -mccabe==0.6.1 # via flake8 -pathtools==0.1.2 # via watchdog -psycopg2==2.7.6.1 -pycodestyle==2.4.0 # via flake8 -pyflakes==2.0.0 # via flake8 -python-dateutil==2.7.5 # via botocore -pytz==2018.7 # via celery -pyyaml==3.13 # via anymarkup, selinon, watchdog -radon==3.0.1 +lxml==4.5.2 # via -r requirements.in, f8a-utils +pyrsistent==0.16.0 # via jsonschema +python-dateutil==2.8.1 # via botocore +pytz==2020.1 # via celery +pyyaml==5.3.1 # via -r requirements.in, anymarkup, selinon rainbow-logging-handler==2.2.2 # via selinon -raven==6.9.0 -requests==2.20.1 -requests-futures==0.9.7 -s3transfer==0.1.13 # via boto3 -semantic-version==2.6.0 -six==1.11.0 # via anymarkup-core, configobj, mando, python-dateutil -sqlalchemy==1.2.14 -toml==0.9.4 # via anymarkup -unidiff==0.5.5 -urllib3==1.24.1 # via botocore, requests -vine==1.1.4 # via amqp -watchdog==0.9.0 -werkzeug==0.14.1 # via flask -xmltodict==0.11.0 # via anymarkup -selinon[celery]==1.0.0 -tenacity==6.2.0 \ No newline at end of file +raven==6.10.0 # via -r requirements.in +requests-futures==1.0.0 # via -r requirements.in +requests==2.24.0 # via -r requirements.in, f8a-utils, requests-futures +s3transfer==0.3.3 # via boto3 +selinon[celery]==1.0.0 # via -r requirements.in +semantic-version==2.8.5 # via -r requirements.in +six==1.15.0 # via anymarkup-core, configobj, jsonschema, pyrsistent, python-dateutil, tenacity +smmap==3.0.4 # via gitdb +soupsieve==2.0.1 # via beautifulsoup4 +sqlalchemy==1.3.18 # via -r requirements.in +tenacity==6.2.0 # via -r requirements.in +toml==0.9.4 # via -r requirements.in, anymarkup +urllib3==1.25.10 # via botocore, requests +vine==1.3.0 # via amqp, celery +werkzeug==1.0.1 # via -r requirements.in +xmltodict==0.12.0 # via anymarkup +zipp==3.1.0 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/setup.py b/setup.py index 00ec73294..7420561fb 100755 --- a/setup.py +++ b/setup.py @@ -7,9 +7,16 @@ def get_requirements(): - """Parse all packages mentioned in the 'requirements.txt' file.""" - with open('requirements.txt') as fd: - return fd.read().splitlines() + """Parse dependencies from 'requirements.in' file.""" + with open('requirements.in') as fd: + lines = fd.read().splitlines() + requires = [] + for line in lines: + requires.append(line) + return requires + + +install_requires = get_requirements() setup( @@ -30,7 +37,7 @@ def get_requirements(): }, packages=find_packages(exclude=['tests', 'tests.*']), include_package_data=True, - install_requires=get_requirements(), + install_requires=install_requires, author='Pavel Odvody', author_email='podvody@redhat.com', description='fabric8-analytics workers & utilities', diff --git a/tests/requirements.in b/tests/requirements.in index c64b5b02e..5d23dd31d 100644 --- a/tests/requirements.in +++ b/tests/requirements.in @@ -1,10 +1,6 @@ -datadiff +pytest<=3.10.1 +radon flexmock -pylint -pytest==3.* -pytest-timeout -pytest-rerunfailures -pytest-cov -pytest-mock -codecov -requests +toml<=0.9.4 +pytest-cov<=2.6.0 +codecov \ No newline at end of file diff --git a/tests/requirements.txt b/tests/requirements.txt index 644e976fa..a3e932042 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -2,31 +2,36 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file requirements.txt requirements.in +# pip-compile # -astroid==2.1.0 # via pylint -atomicwrites==1.2.1 # via pytest -attrs==18.2.0 # via pytest -certifi==2018.10.15 # via requests +atomicwrites==1.4.0 # via pytest +attrs==19.3.0 # via pytest +certifi==2020.6.20 # via requests chardet==3.0.4 # via requests -codecov==2.0.15 -coverage==4.5.2 # via codecov, pytest-cov -datadiff==2.0.0 -flexmock==0.10.2 -idna==2.7 # via requests -isort==4.3.4 # via pylint -lazy-object-proxy==1.3.1 # via astroid -mccabe==0.6.1 # via pylint -more-itertools==4.3.0 # via pytest -pluggy==0.8.0 # via pytest -py==1.7.0 # via pytest -pylint==2.2.2 -pytest-cov==2.6.0 -pytest-mock==1.10.0 -pytest-rerunfailures==5.0 -pytest-timeout==1.3.3 -pytest==3.10.1 -requests==2.20.1 -six==1.11.0 # via astroid, more-itertools, pytest -urllib3==1.24.1 # via requests -wrapt==1.10.11 # via astroid +codecov==2.1.8 # via -r requirements.in +colorama==0.4.3 # via radon +coverage==5.2.1 # via codecov, pytest-cov +flake8-polyfill==1.0.2 # via radon +flake8==3.8.3 # via flake8-polyfill +flexmock==0.10.4 # via -r requirements.in +future==0.18.2 # via radon +idna==2.10 # via requests +importlib-metadata==1.7.0 # via flake8, pluggy +mando==0.6.4 # via radon +mccabe==0.6.1 # via flake8 +more-itertools==8.4.0 # via pytest +pluggy==0.13.1 # via pytest +py==1.9.0 # via pytest +pycodestyle==2.6.0 # via flake8 +pyflakes==2.2.0 # via flake8 +pytest-cov==2.6.0 # via -r requirements.in +pytest==3.10.1 # via -r requirements.in, pytest-cov +radon==4.2.0 # via -r requirements.in +requests==2.24.0 # via codecov +six==1.15.0 # via mando, pytest +toml==0.9.4 # via -r requirements.in +urllib3==1.25.10 # via requests +zipp==3.1.0 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools