Merge pull request #6325 from mvdbeek/py3_fixes2

 Add py34-first_startup to Travis and default tox tests

.travis.yml

@@ -14,6 +14,15 @@ env:
 
 matrix:
   include:
+    - env: TOX_ENV=py34-first_startup
+      addons:
+        apt:
+          packages:
+            # For psutil, pyyaml, uwsgi...
+            - libpython3.4-dev
+            # For python-lzo
+            - liblzo2-dev
+            - zlib1g-dev
     - env: TOX_ENV=validate_test_tools
       addons:
         apt:

config/galaxy.yml.sample

@@ -475,19 +475,19 @@ galaxy:
   #default_job_shell: /bin/bash
 
   # Citation related caching.  Tool citations information maybe fetched
-  # from external sources such as http://dx.doi.org/ by Galaxy - the
+  # from external sources such as https://doi.org/ by Galaxy - the
   # following parameters can be used to control the caching used to
   # store this information.
   #citation_cache_type: file
 
   # Citation related caching.  Tool citations information maybe fetched
-  # from external sources such as http://dx.doi.org/ by Galaxy - the
+  # from external sources such as https://doi.org/ by Galaxy - the
   # following parameters can be used to control the caching used to
   # store this information.
   #citation_cache_data_dir: database/citations/data
 
   # Citation related caching.  Tool citations information maybe fetched
-  # from external sources such as http://dx.doi.org/ by Galaxy - the
+  # from external sources such as https://doi.org/ by Galaxy - the
   # following parameters can be used to control the caching used to
   # store this information.
   #citation_cache_lock_dir: database/citations/lock
@@ -1178,10 +1178,10 @@ galaxy:
   # Galaxy encodes various internal values when these values will be
   # output in some format (for example, in a URL or cookie).  You should
   # set a key to be used by the algorithm that encodes and decodes these
-  # values.  It can be any string up to 448 bits long. One simple way to
-  # generate a value for this is with the shell command:   python -c
-  # 'from __future__ import print_function; import time;
-  # print(time.time())' | md5sum | cut -f 1 -d ' '
+  # values. It can be any string with a length between 5 and 56 bytes.
+  # One simple way to generate a value for this is with the shell
+  # command:   python -c 'from __future__ import print_function; import
+  # time; print(time.time())' | md5sum | cut -f 1 -d ' '
   #id_secret: USING THE DEFAULT IS NOT SECURE!
 
   # User authentication can be delegated to an upstream proxy server

config/tool_shed.yml.sample

@@ -323,19 +323,19 @@ tool_shed:
   #brand: null
 
   # Citation related caching.  Tool citations information maybe fetched
-  # from external sources such as http://dx.doi.org/ by Galaxy - the
+  # from external sources such as https://doi.org/ by Galaxy - the
   # following parameters can be used to control the caching used to
   # store this information.
   #citation_cache_type: file
 
   # Citation related caching.  Tool citations information maybe fetched
-  # from external sources such as http://dx.doi.org/ by Galaxy - the
+  # from external sources such as https://doi.org/ by Galaxy - the
   # following parameters can be used to control the caching used to
   # store this information.
   #citation_cache_data_dir: database/citations/data
 
   # Citation related caching.  Tool citations information maybe fetched
-  # from external sources such as http://dx.doi.org/ by Galaxy - the
+  # from external sources such as https://doi.org/ by Galaxy - the
   # following parameters can be used to control the caching used to
   # store this information.
   #citation_cache_lock_dir: database/citations/lock

doc/source/admin/galaxy_options.rst

@@ -846,8 +846,8 @@
 
 :Description:
     Citation related caching.  Tool citations information maybe
-    fetched from external sources such as https://doi.org/ by Galaxy
-    - the following parameters can be used to control the caching used
+    fetched from external sources such as https://doi.org/ by Galaxy -
+    the following parameters can be used to control the caching used
     to store this information.
 :Default: ``file``
 :Type: str
@@ -859,8 +859,8 @@
 
 :Description:
     Citation related caching.  Tool citations information maybe
-    fetched from external sources such as https://doi.org/ by Galaxy
-    - the following parameters can be used to control the caching used
+    fetched from external sources such as https://doi.org/ by Galaxy -
+    the following parameters can be used to control the caching used
     to store this information.
 :Default: ``database/citations/data``
 :Type: str
@@ -872,8 +872,8 @@
 
 :Description:
     Citation related caching.  Tool citations information maybe
-    fetched from external sources such as https://doi.org/ by Galaxy
-    - the following parameters can be used to control the caching used
+    fetched from external sources such as https://doi.org/ by Galaxy -
+    the following parameters can be used to control the caching used
     to store this information.
 :Default: ``database/citations/lock``
 :Type: str
@@ -2440,10 +2440,11 @@
     Galaxy encodes various internal values when these values will be
     output in some format (for example, in a URL or cookie).  You
     should set a key to be used by the algorithm that encodes and
-    decodes these values.  It can be any string up to 448 bits long.
-    One simple way to generate a value for this is with the shell
-    command:   python -c 'from __future__ import print_function;
-    import time; print(time.time())' | md5sum | cut -f 1 -d ' '
+    decodes these values. It can be any string with a length between 5
+    and 56 bytes. One simple way to generate a value for this is with
+    the shell command:   python -c 'from __future__ import
+    print_function; import time; print(time.time())' | md5sum | cut -f
+    1 -d ' '
 :Default: ``USING THE DEFAULT IS NOT SECURE!``
 :Type: str
 

lib/galaxy/datatypes/display_applications/util.py

@@ -1,5 +1,7 @@
 from Crypto.Cipher import Blowfish
 
+from galaxy.util import smart_str
+
 
 def encode_dataset_user(trans, dataset, user):
     # encode dataset id as usual
@@ -11,7 +13,7 @@ def encode_dataset_user(trans, dataset, user):
         user_hash = str(user.id)
         # Pad to a multiple of 8 with leading "!"
         user_hash = ("!" * (8 - len(user_hash) % 8)) + user_hash
-        cipher = Blowfish.new(str(dataset.create_time))
+        cipher = Blowfish.new(smart_str(dataset.create_time), mode=Blowfish.MODE_ECB)
         user_hash = cipher.encrypt(user_hash).encode('hex')
     return dataset_hash, user_hash
 
@@ -25,7 +27,7 @@ def decode_dataset_user(trans, dataset_hash, user_hash):
     if user_hash in [None, 'None']:
         user = None
     else:
-        cipher = Blowfish.new(str(dataset.create_time))
+        cipher = Blowfish.new(smart_str(dataset.create_time), mode=Blowfish.MODE_ECB)
         user_id = cipher.decrypt(user_hash.decode('hex')).lstrip("!")
         user = trans.sa_session.query(trans.app.model.User).get(int(user_id))
         assert user, "A Bad user id was passed to decode_dataset_user"

lib/galaxy/dependencies/pipfiles/default/Pipfile

@@ -13,7 +13,7 @@ PyYAML = "*"
 SQLAlchemy = "*"
 SQLAlchemy-Utils = "*"
 Mercurial = {version = "<=3.7.3", markers = "python_version < '3'"}
-pycrypto = "*"
+pycryptodome = "*"
 uWSGI = "*"
 pysam = "==0.14.1"
 bdbag = "*"

lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt

@@ -4,7 +4,7 @@ asn1crypto==0.24.0
 babel==2.5.3
 bagit==1.6.4
 bcrypt==3.1.4
-bdbag==1.2.4
+bdbag==1.3.0
 beaker==1.9.1
 bioblend==0.10.0
 bleach==2.1.3
@@ -55,7 +55,7 @@ monotonic==1.4
 msgpack==0.5.6
 munch==2.3.1
 netaddr==0.7.19
-netifaces==0.10.6
+netifaces==0.10.7
 nose==1.3.7
 numpy==1.14.2
 oauthlib==2.0.7
@@ -81,8 +81,8 @@ pulsar-galaxy-lib==0.8.3
 py2-ipaddress==3.4.1; python_version < '3'
 pyasn1==0.4.2
 pycparser==2.18
-pycrypto==2.6.1
-pycryptodomex==3.6.0
+pycryptodome==3.6.1
+pycryptodomex==3.6.1
 pyjwkest==1.4.0
 pyjwt==1.6.1
 pykwalify==1.6.1

lib/galaxy/web/security/__init__.py

@@ -23,7 +23,7 @@ class SecurityHelper(object):
     def __init__(self, **config):
         id_secret = config['id_secret']
         self.id_secret = id_secret
-        self.id_cipher = Blowfish.new(self.id_secret)
+        self.id_cipher = Blowfish.new(smart_str(self.id_secret), mode=Blowfish.MODE_ECB)
 
         per_kind_id_secret_base = config.get('per_kind_id_secret_base', self.id_secret)
         self.id_ciphers_for_kind = _cipher_cache(per_kind_id_secret_base)
@@ -116,7 +116,7 @@ def __init__(self, secret_base):
     def __missing__(self, key):
         assert len(key) < 15, KIND_TOO_LONG_MESSAGE
         secret = self.secret_base + "__" + key
-        return Blowfish.new(_last_bits(secret))
+        return Blowfish.new(_last_bits(secret), mode=Blowfish.MODE_ECB)
 
 
 def _last_bits(secret):

lib/galaxy/webapps/galaxy/config_schema.yml

@@ -1821,8 +1821,8 @@ mapping:
         desc: |
           Galaxy encodes various internal values when these values will be output in
           some format (for example, in a URL or cookie).  You should set a key to be
-          used by the algorithm that encodes and decodes these values.  It can be any
-          string up to 448 bits long.
+          used by the algorithm that encodes and decodes these values. It can be any
+          string with a length between 5 and 56 bytes.
           One simple way to generate a value for this is with the shell command:
             python -c 'from __future__ import print_function; import time; print(time.time())' | md5sum | cut -f 1 -d ' '
 

lib/tool_shed/util/common_util.py

@@ -164,7 +164,7 @@ def get_repository_dependencies(app, tool_shed_url, repository_name, repository_
         log.warning("The URL\n%s\nraised the exception:\n%s\n", util.build_url(tool_shed_url, pathspec=pathspec, params=params), e)
     if tool_shed_accessible:
         if len(raw_text) > 2:
-            encoded_text = json.loads(raw_text)
+            encoded_text = json.loads(util.unicodify(raw_text))
             repository_dependencies_dict = encoding_util.tool_shed_decode(encoded_text)
     return tool_shed_accessible, repository_dependencies_dict
 

lib/tool_shed/util/encoding_util.py

@@ -2,6 +2,7 @@
 import json
 import logging
 
+from galaxy.util import unicodify
 from galaxy.util.hash_util import hmac_new
 
 log = logging.getLogger(__name__)
@@ -12,12 +13,14 @@
 
 def tool_shed_decode(value):
     # Extract and verify hash
+    value = unicodify(value)
     a, b = value.split(":")
     value = binascii.unhexlify(b)
-    test = hmac_new('ToolShedAndGalaxyMustHaveThisSameKey', value)
+    test = hmac_new(b'ToolShedAndGalaxyMustHaveThisSameKey', value)
     assert a == test
     # Restore from string
     values = None
+    value = unicodify(value)
     try:
         values = json.loads(value)
     except Exception:

lib/tool_shed/util/hg_util.py

@@ -5,11 +5,6 @@
 from datetime import datetime
 from time import gmtime
 
-from mercurial import (
-    hg,
-    ui
-)
-
 from tool_shed.util import basic_util
 
 log = logging.getLogger(__name__)
@@ -201,6 +196,11 @@ def get_readable_ctx_date(ctx):
 
 
 def get_repo_for_repository(app, repository=None, repo_path=None):
+    # Import from mercurial here to let Galaxy start under Python 3
+    from mercurial import (
+        hg,
+        ui
+    )
     if repository is not None:
         return hg.repository(ui.ui(), repository.repo_path(app))
     if repo_path is not None:

scripts/api/common.py

@@ -7,7 +7,6 @@
 import logging
 import sys
 
-from Crypto.Cipher import Blowfish
 from six.moves.urllib.error import HTTPError
 from six.moves.urllib.request import Request, urlopen
 
@@ -185,16 +184,3 @@ def delete(api_key, url, data, return_formatted=True):
     print('Response')
     print('--------')
     print(r)
-
-
-def encode_id(config_id_secret, obj_id):
-    """
-    utility method to encode ID's
-    """
-    id_cipher = Blowfish.new(config_id_secret)
-    # Convert to string
-    s = str(obj_id)
-    # Pad to a multiple of 8 with leading "!"
-    s = ("!" * (8 - len(s) % 8)) + s
-    # Encrypt
-    return id_cipher.encrypt(s).encode('hex')

scripts/check_python.py

@@ -1,30 +1,43 @@
 """
-If the current installed python version is not 2.7, prints an error
+If the current installed Python version is not supported, prints an error
 message to stderr and returns 1
 """
 from __future__ import print_function
 
+import os
 import sys
 
+version_string = '.'.join(str(_) for _ in sys.version_info[:3])
+
 msg = """ERROR: Your Python version is: %s
-Galaxy is currently supported on Python 2.7 only.  To run Galaxy,
-please download and install a supported version from python.org.  If a
-supported version is installed but is not your default, getgalaxy.org
-contains instructions on how to force Galaxy to use a different version.""" % sys.version[:3]
+Galaxy is currently supported on Python 2.7 only. To run Galaxy, please
+install a supported Python version. If a supported version is already
+installed but is not your default, https://galaxyproject.org/admin/python/
+contains instructions on how to force Galaxy to use a different version.""" % version_string
+
+msg_beta = """WARNING: Your Python version is: %s
+Galaxy is currently supported on Python 2.7 only, support for Python >= 3.4
+is still in beta stage. Since you are using a virtual environment, we assume
+you are developing or testing Galaxy under Python 3. If not,
+install a supported Python version. If a supported version is already
+installed but is not your default, https://galaxyproject.org/admin/python/
+contains instructions on how to force Galaxy to use a different version.""" % version_string
 
 
 def check_python():
-    try:
-        assert sys.version_info[:2] == (2, 7)
-    except AssertionError:
+    venv = os.getenv('VIRTUAL_ENV')
+    if sys.version_info[:2] == (2, 7):
+        # supported
+        return
+    elif sys.version_info[:2] >= (3, 4) and venv:
+        print(msg_beta, file=sys.stderr)
+    else:
         print(msg, file=sys.stderr)
-        raise
+        raise Exception(msg)
 
 
 if __name__ == '__main__':
-    rval = 0
     try:
         check_python()
-    except Exception:
-        rval = 1
-    sys.exit(rval)
+    except Exception as e:
+        sys.exit(1)

test/unit/test_security_helper.py

@@ -3,8 +3,8 @@
 from galaxy.web import security
 
 
-test_helper_1 = security.SecurityHelper(id_secret="sec1")
-test_helper_2 = security.SecurityHelper(id_secret="sec2")
+test_helper_1 = security.SecurityHelper(id_secret="secu1")
+test_helper_2 = security.SecurityHelper(id_secret="secu2")
 
 
 def test_maximum_length_handling_ascii():

test/unit/unittest_utils/galaxy_mock.py

@@ -103,7 +103,7 @@ class MockAppConfig(Bunch):
     def __init__(self, root=None, **kwargs):
         Bunch.__init__(self, **kwargs)
         root = root or '/tmp'
-        self.security = security.SecurityHelper(id_secret='bler')
+        self.security = security.SecurityHelper(id_secret='6e46ed6483a833c100e68cc3f1d0dd76')
         self.use_remote_user = kwargs.get('use_remote_user', False)
         self.file_path = '/tmp'
         self.jobs_directory = '/tmp'
@@ -142,7 +142,7 @@ class MockWebapp(object):
 
     def __init__(self, **kwargs):
         self.name = kwargs.get('name', 'galaxy')
-        self.security = security.SecurityHelper(id_secret='bler')
+        self.security = security.SecurityHelper(id_secret='6e46ed6483a833c100e68cc3f1d0dd76')
 
 
 class MockTrans(object):