diff --git a/.travis.yml b/.travis.yml index 81d6ff5d5409..2123c89d519f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,15 @@ env: matrix: include: + - env: TOX_ENV=py34-first_startup + addons: + apt: + packages: + # For psutil, pyyaml, uwsgi... + - libpython3.4-dev + # For python-lzo + - liblzo2-dev + - zlib1g-dev - env: TOX_ENV=validate_test_tools addons: apt: diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample index 0fdca54e53e6..4a047f06e1fd 100644 --- a/config/galaxy.yml.sample +++ b/config/galaxy.yml.sample @@ -475,19 +475,19 @@ galaxy: #default_job_shell: /bin/bash # Citation related caching. Tool citations information maybe fetched - # from external sources such as http://dx.doi.org/ by Galaxy - the + # from external sources such as https://doi.org/ by Galaxy - the # following parameters can be used to control the caching used to # store this information. #citation_cache_type: file # Citation related caching. Tool citations information maybe fetched - # from external sources such as http://dx.doi.org/ by Galaxy - the + # from external sources such as https://doi.org/ by Galaxy - the # following parameters can be used to control the caching used to # store this information. #citation_cache_data_dir: database/citations/data # Citation related caching. Tool citations information maybe fetched - # from external sources such as http://dx.doi.org/ by Galaxy - the + # from external sources such as https://doi.org/ by Galaxy - the # following parameters can be used to control the caching used to # store this information. #citation_cache_lock_dir: database/citations/lock @@ -1178,10 +1178,10 @@ galaxy: # Galaxy encodes various internal values when these values will be # output in some format (for example, in a URL or cookie). You should # set a key to be used by the algorithm that encodes and decodes these - # values. It can be any string up to 448 bits long. One simple way to - # generate a value for this is with the shell command: python -c - # 'from __future__ import print_function; import time; - # print(time.time())' | md5sum | cut -f 1 -d ' ' + # values. It can be any string with a length between 5 and 56 bytes. + # One simple way to generate a value for this is with the shell + # command: python -c 'from __future__ import print_function; import + # time; print(time.time())' | md5sum | cut -f 1 -d ' ' #id_secret: USING THE DEFAULT IS NOT SECURE! # User authentication can be delegated to an upstream proxy server diff --git a/config/tool_shed.yml.sample b/config/tool_shed.yml.sample index 986d4be6c556..d3fffe4806aa 100644 --- a/config/tool_shed.yml.sample +++ b/config/tool_shed.yml.sample @@ -323,19 +323,19 @@ tool_shed: #brand: null # Citation related caching. Tool citations information maybe fetched - # from external sources such as http://dx.doi.org/ by Galaxy - the + # from external sources such as https://doi.org/ by Galaxy - the # following parameters can be used to control the caching used to # store this information. #citation_cache_type: file # Citation related caching. Tool citations information maybe fetched - # from external sources such as http://dx.doi.org/ by Galaxy - the + # from external sources such as https://doi.org/ by Galaxy - the # following parameters can be used to control the caching used to # store this information. #citation_cache_data_dir: database/citations/data # Citation related caching. Tool citations information maybe fetched - # from external sources such as http://dx.doi.org/ by Galaxy - the + # from external sources such as https://doi.org/ by Galaxy - the # following parameters can be used to control the caching used to # store this information. #citation_cache_lock_dir: database/citations/lock diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index e150d8876453..ca3e2945c24a 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -846,8 +846,8 @@ :Description: Citation related caching. Tool citations information maybe - fetched from external sources such as https://doi.org/ by Galaxy - - the following parameters can be used to control the caching used + fetched from external sources such as https://doi.org/ by Galaxy - + the following parameters can be used to control the caching used to store this information. :Default: ``file`` :Type: str @@ -859,8 +859,8 @@ :Description: Citation related caching. Tool citations information maybe - fetched from external sources such as https://doi.org/ by Galaxy - - the following parameters can be used to control the caching used + fetched from external sources such as https://doi.org/ by Galaxy - + the following parameters can be used to control the caching used to store this information. :Default: ``database/citations/data`` :Type: str @@ -872,8 +872,8 @@ :Description: Citation related caching. Tool citations information maybe - fetched from external sources such as https://doi.org/ by Galaxy - - the following parameters can be used to control the caching used + fetched from external sources such as https://doi.org/ by Galaxy - + the following parameters can be used to control the caching used to store this information. :Default: ``database/citations/lock`` :Type: str @@ -2440,10 +2440,11 @@ Galaxy encodes various internal values when these values will be output in some format (for example, in a URL or cookie). You should set a key to be used by the algorithm that encodes and - decodes these values. It can be any string up to 448 bits long. - One simple way to generate a value for this is with the shell - command: python -c 'from __future__ import print_function; - import time; print(time.time())' | md5sum | cut -f 1 -d ' ' + decodes these values. It can be any string with a length between 5 + and 56 bytes. One simple way to generate a value for this is with + the shell command: python -c 'from __future__ import + print_function; import time; print(time.time())' | md5sum | cut -f + 1 -d ' ' :Default: ``USING THE DEFAULT IS NOT SECURE!`` :Type: str diff --git a/lib/galaxy/datatypes/display_applications/util.py b/lib/galaxy/datatypes/display_applications/util.py index f71eb5ed6390..81f2f3818ef8 100644 --- a/lib/galaxy/datatypes/display_applications/util.py +++ b/lib/galaxy/datatypes/display_applications/util.py @@ -1,5 +1,7 @@ from Crypto.Cipher import Blowfish +from galaxy.util import smart_str + def encode_dataset_user(trans, dataset, user): # encode dataset id as usual @@ -11,7 +13,7 @@ def encode_dataset_user(trans, dataset, user): user_hash = str(user.id) # Pad to a multiple of 8 with leading "!" user_hash = ("!" * (8 - len(user_hash) % 8)) + user_hash - cipher = Blowfish.new(str(dataset.create_time)) + cipher = Blowfish.new(smart_str(dataset.create_time), mode=Blowfish.MODE_ECB) user_hash = cipher.encrypt(user_hash).encode('hex') return dataset_hash, user_hash @@ -25,7 +27,7 @@ def decode_dataset_user(trans, dataset_hash, user_hash): if user_hash in [None, 'None']: user = None else: - cipher = Blowfish.new(str(dataset.create_time)) + cipher = Blowfish.new(smart_str(dataset.create_time), mode=Blowfish.MODE_ECB) user_id = cipher.decrypt(user_hash.decode('hex')).lstrip("!") user = trans.sa_session.query(trans.app.model.User).get(int(user_id)) assert user, "A Bad user id was passed to decode_dataset_user" diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index b409374f7fd6..d78ecf871e62 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -13,7 +13,7 @@ PyYAML = "*" SQLAlchemy = "*" SQLAlchemy-Utils = "*" Mercurial = {version = "<=3.7.3", markers = "python_version < '3'"} -pycrypto = "*" +pycryptodome = "*" uWSGI = "*" pysam = "==0.14.1" bdbag = "*" diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 5ed36dd808c2..ff701d3cea9e 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -4,7 +4,7 @@ asn1crypto==0.24.0 babel==2.5.3 bagit==1.6.4 bcrypt==3.1.4 -bdbag==1.2.4 +bdbag==1.3.0 beaker==1.9.1 bioblend==0.10.0 bleach==2.1.3 @@ -55,7 +55,7 @@ monotonic==1.4 msgpack==0.5.6 munch==2.3.1 netaddr==0.7.19 -netifaces==0.10.6 +netifaces==0.10.7 nose==1.3.7 numpy==1.14.2 oauthlib==2.0.7 @@ -81,8 +81,8 @@ pulsar-galaxy-lib==0.8.3 py2-ipaddress==3.4.1; python_version < '3' pyasn1==0.4.2 pycparser==2.18 -pycrypto==2.6.1 -pycryptodomex==3.6.0 +pycryptodome==3.6.1 +pycryptodomex==3.6.1 pyjwkest==1.4.0 pyjwt==1.6.1 pykwalify==1.6.1 diff --git a/lib/galaxy/dependencies/pipfiles/flake8/Pipfile.lock b/lib/galaxy/dependencies/pipfiles/flake8/Pipfile.lock index e569475cfca6..c7c2dc7816a4 100644 --- a/lib/galaxy/dependencies/pipfiles/flake8/Pipfile.lock +++ b/lib/galaxy/dependencies/pipfiles/flake8/Pipfile.lock @@ -56,8 +56,6 @@ }, "pycodestyle": { "hashes": [ - "sha256:1ec08a51c901dfe44921576ed6e4c1f5b7ecbad403f871397feedb5eb8e4fa14", - "sha256:5ff2fbcbab997895ba9ead77e1b38b3ebc2e5c3b8a6194ef918666e4c790a00e", "sha256:682256a5b318149ca0d2a9185d365d8864a768a28db66a84a2ea946bcc426766", "sha256:6c4245ade1edfad79c3446fadfc96b0de2759662dc29d07d80a6f27ad1ca6ba9" ], diff --git a/lib/galaxy/web/security/__init__.py b/lib/galaxy/web/security/__init__.py index 82b8ea06bc9f..74ecb862f229 100644 --- a/lib/galaxy/web/security/__init__.py +++ b/lib/galaxy/web/security/__init__.py @@ -23,7 +23,7 @@ class SecurityHelper(object): def __init__(self, **config): id_secret = config['id_secret'] self.id_secret = id_secret - self.id_cipher = Blowfish.new(self.id_secret) + self.id_cipher = Blowfish.new(smart_str(self.id_secret), mode=Blowfish.MODE_ECB) per_kind_id_secret_base = config.get('per_kind_id_secret_base', self.id_secret) self.id_ciphers_for_kind = _cipher_cache(per_kind_id_secret_base) @@ -116,7 +116,7 @@ def __init__(self, secret_base): def __missing__(self, key): assert len(key) < 15, KIND_TOO_LONG_MESSAGE secret = self.secret_base + "__" + key - return Blowfish.new(_last_bits(secret)) + return Blowfish.new(_last_bits(secret), mode=Blowfish.MODE_ECB) def _last_bits(secret): diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index a05fd8fec608..aa9957354453 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -1821,8 +1821,8 @@ mapping: desc: | Galaxy encodes various internal values when these values will be output in some format (for example, in a URL or cookie). You should set a key to be - used by the algorithm that encodes and decodes these values. It can be any - string up to 448 bits long. + used by the algorithm that encodes and decodes these values. It can be any + string with a length between 5 and 56 bytes. One simple way to generate a value for this is with the shell command: python -c 'from __future__ import print_function; import time; print(time.time())' | md5sum | cut -f 1 -d ' ' diff --git a/lib/tool_shed/util/common_util.py b/lib/tool_shed/util/common_util.py index 9063dd0247c0..39b1e1c90122 100644 --- a/lib/tool_shed/util/common_util.py +++ b/lib/tool_shed/util/common_util.py @@ -164,7 +164,7 @@ def get_repository_dependencies(app, tool_shed_url, repository_name, repository_ log.warning("The URL\n%s\nraised the exception:\n%s\n", util.build_url(tool_shed_url, pathspec=pathspec, params=params), e) if tool_shed_accessible: if len(raw_text) > 2: - encoded_text = json.loads(raw_text) + encoded_text = json.loads(util.unicodify(raw_text)) repository_dependencies_dict = encoding_util.tool_shed_decode(encoded_text) return tool_shed_accessible, repository_dependencies_dict diff --git a/lib/tool_shed/util/encoding_util.py b/lib/tool_shed/util/encoding_util.py index a31e5149d09c..b98b371c9308 100644 --- a/lib/tool_shed/util/encoding_util.py +++ b/lib/tool_shed/util/encoding_util.py @@ -2,6 +2,7 @@ import json import logging +from galaxy.util import unicodify from galaxy.util.hash_util import hmac_new log = logging.getLogger(__name__) @@ -12,12 +13,14 @@ def tool_shed_decode(value): # Extract and verify hash + value = unicodify(value) a, b = value.split(":") value = binascii.unhexlify(b) - test = hmac_new('ToolShedAndGalaxyMustHaveThisSameKey', value) + test = hmac_new(b'ToolShedAndGalaxyMustHaveThisSameKey', value) assert a == test # Restore from string values = None + value = unicodify(value) try: values = json.loads(value) except Exception: diff --git a/lib/tool_shed/util/hg_util.py b/lib/tool_shed/util/hg_util.py index 98ced5672419..5440a237ce72 100644 --- a/lib/tool_shed/util/hg_util.py +++ b/lib/tool_shed/util/hg_util.py @@ -5,11 +5,6 @@ from datetime import datetime from time import gmtime -from mercurial import ( - hg, - ui -) - from tool_shed.util import basic_util log = logging.getLogger(__name__) @@ -201,6 +196,11 @@ def get_readable_ctx_date(ctx): def get_repo_for_repository(app, repository=None, repo_path=None): + # Import from mercurial here to let Galaxy start under Python 3 + from mercurial import ( + hg, + ui + ) if repository is not None: return hg.repository(ui.ui(), repository.repo_path(app)) if repo_path is not None: diff --git a/scripts/api/common.py b/scripts/api/common.py index 440808d88dd9..646499cf5f1e 100644 --- a/scripts/api/common.py +++ b/scripts/api/common.py @@ -7,7 +7,6 @@ import logging import sys -from Crypto.Cipher import Blowfish from six.moves.urllib.error import HTTPError from six.moves.urllib.request import Request, urlopen @@ -185,16 +184,3 @@ def delete(api_key, url, data, return_formatted=True): print('Response') print('--------') print(r) - - -def encode_id(config_id_secret, obj_id): - """ - utility method to encode ID's - """ - id_cipher = Blowfish.new(config_id_secret) - # Convert to string - s = str(obj_id) - # Pad to a multiple of 8 with leading "!" - s = ("!" * (8 - len(s) % 8)) + s - # Encrypt - return id_cipher.encrypt(s).encode('hex') diff --git a/scripts/check_python.py b/scripts/check_python.py index 40e22d27174a..91442e10d5e8 100644 --- a/scripts/check_python.py +++ b/scripts/check_python.py @@ -1,30 +1,43 @@ """ -If the current installed python version is not 2.7, prints an error +If the current installed Python version is not supported, prints an error message to stderr and returns 1 """ from __future__ import print_function +import os import sys +version_string = '.'.join(str(_) for _ in sys.version_info[:3]) + msg = """ERROR: Your Python version is: %s -Galaxy is currently supported on Python 2.7 only. To run Galaxy, -please download and install a supported version from python.org. If a -supported version is installed but is not your default, getgalaxy.org -contains instructions on how to force Galaxy to use a different version.""" % sys.version[:3] +Galaxy is currently supported on Python 2.7 only. To run Galaxy, please +install a supported Python version. If a supported version is already +installed but is not your default, https://galaxyproject.org/admin/python/ +contains instructions on how to force Galaxy to use a different version.""" % version_string + +msg_beta = """WARNING: Your Python version is: %s +Galaxy is currently supported on Python 2.7 only, support for Python >= 3.4 +is still in beta stage. Since you are using a virtual environment, we assume +you are developing or testing Galaxy under Python 3. If not, +install a supported Python version. If a supported version is already +installed but is not your default, https://galaxyproject.org/admin/python/ +contains instructions on how to force Galaxy to use a different version.""" % version_string def check_python(): - try: - assert sys.version_info[:2] == (2, 7) - except AssertionError: + venv = os.getenv('VIRTUAL_ENV') + if sys.version_info[:2] == (2, 7): + # supported + return + elif sys.version_info[:2] >= (3, 4) and venv: + print(msg_beta, file=sys.stderr) + else: print(msg, file=sys.stderr) - raise + raise Exception(msg) if __name__ == '__main__': - rval = 0 try: check_python() - except Exception: - rval = 1 - sys.exit(rval) + except Exception as e: + sys.exit(1) diff --git a/test/unit/test_security_helper.py b/test/unit/test_security_helper.py index 6e39cfc86377..8f96dd3fcb91 100644 --- a/test/unit/test_security_helper.py +++ b/test/unit/test_security_helper.py @@ -3,8 +3,8 @@ from galaxy.web import security -test_helper_1 = security.SecurityHelper(id_secret="sec1") -test_helper_2 = security.SecurityHelper(id_secret="sec2") +test_helper_1 = security.SecurityHelper(id_secret="secu1") +test_helper_2 = security.SecurityHelper(id_secret="secu2") def test_maximum_length_handling_ascii(): diff --git a/test/unit/unittest_utils/galaxy_mock.py b/test/unit/unittest_utils/galaxy_mock.py index 7ea413590e46..5464530da43f 100644 --- a/test/unit/unittest_utils/galaxy_mock.py +++ b/test/unit/unittest_utils/galaxy_mock.py @@ -103,7 +103,7 @@ class MockAppConfig(Bunch): def __init__(self, root=None, **kwargs): Bunch.__init__(self, **kwargs) root = root or '/tmp' - self.security = security.SecurityHelper(id_secret='bler') + self.security = security.SecurityHelper(id_secret='6e46ed6483a833c100e68cc3f1d0dd76') self.use_remote_user = kwargs.get('use_remote_user', False) self.file_path = '/tmp' self.jobs_directory = '/tmp' @@ -142,7 +142,7 @@ class MockWebapp(object): def __init__(self, **kwargs): self.name = kwargs.get('name', 'galaxy') - self.security = security.SecurityHelper(id_secret='bler') + self.security = security.SecurityHelper(id_secret='6e46ed6483a833c100e68cc3f1d0dd76') class MockTrans(object): diff --git a/tox.ini b/tox.ini index 1844ef98405c..9f51bcfbd891 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] # envlist is the list of environments that are tested when `tox` is run without any option # hyphens in an environment name are used to delimit factors -envlist = check_py3_compatibility, py27-first_startup, py27-lint, py27-lint_docstring_include_list, py27-unit, py34-lint, qunit, validate_test_tools +envlist = check_py3_compatibility, py27-first_startup, py27-lint, py27-lint_docstring_include_list, py27-unit, py34-lint, py34-first_startup, qunit, validate_test_tools skipsdist = True [testenv:check_py3_compatibility]