[20.01] Backport 9175: Update galaxy.tool_util.cwl for latest changes in CWL branch.

.github/workflows/mulled.yaml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-18.04
     strategy:
       matrix:
-        python-version: [2.7, 3.7]
+        python-version: [3.7]
     steps:
     - name: Cache tox dir
       uses: actions/cache@v1

lib/galaxy/config/config_manage.py

@@ -28,7 +28,10 @@
 
 
 from galaxy.config import GALAXY_CONFIG_SCHEMA_PATH
-from galaxy.config.schema import AppSchema, Schema
+from galaxy.config.schema import (
+    AppSchema,
+    Schema,
+)
 from galaxy.util import safe_makedirs
 from galaxy.util.properties import nice_config_parser
 from galaxy.util.yaml_util import (
@@ -373,7 +376,7 @@ def main(argv=None):
         argv = sys.argv[1:]
     args = _arg_parser().parse_args(argv)
     app_name = args.app
-    app_desc = APPS.get(app_name, None)
+    app_desc = APPS.get(app_name)
     action = args.action
     action_func = ACTIONS[action]
     action_func(args, app_desc)
@@ -520,31 +523,29 @@ def _lint(args, app_desc):
 
 
 def _validate(args, app_desc):
+    if Core is None:
+        raise Exception("Cannot validate file, pykwalify is not installed.")
     path = _find_config(args, app_desc)
-    # Allow empty mapping (not allowed by pykawlify)
+    # Allow empty mapping (not allowed by pykwalify)
     raw_config = _order_load_path(path)
-    if raw_config.get(app_desc.app_name, None) is None:
+    if raw_config.get(app_desc.app_name) is None:
         raw_config[app_desc.app_name] = {}
-        config_p = tempfile.NamedTemporaryFile('w', delete=False, suffix=".yml")
+    # Rewrite the file any way to merge any duplicate keys
+    with tempfile.NamedTemporaryFile('w', delete=False, suffix=".yml") as config_p:
         ordered_dump(raw_config, config_p)
-        config_p.flush()
-        path = config_p.name
-
-    fp = tempfile.NamedTemporaryFile('w', delete=False, suffix=".yml")
 
     def _clean(p, k, v):
         return k not in ['reloadable', 'path_resolves_to']
 
     clean_schema = remap(app_desc.schema.raw_schema, _clean)
-    ordered_dump(clean_schema, fp)
-    fp.flush()
-    name = fp.name
-    if Core is None:
-        raise Exception("Cannot validate file, pykwalify is not installed.")
-    c = Core(
-        source_file=path,
-        schema_files=[name],
-    )
+    with tempfile.NamedTemporaryFile('w', suffix=".yml") as fp:
+        ordered_dump(clean_schema, fp)
+        fp.flush()
+        c = Core(
+            source_file=config_p.name,
+            schema_files=[fp.name],
+        )
+    os.remove(config_p.name)
     c.validate()
 
 
@@ -713,8 +714,8 @@ def _write_to_file(args, f, path):
 def _order_load_path(path):
     """Load (with ``_ordered_load``) on specified path (a YAML file)."""
     with open(path, "r") as f:
-        # Allow empty mapping (not allowed by pykawlify)
-        raw_config = ordered_load(f)
+        # Allow empty mapping (not allowed by pykwalify)
+        raw_config = ordered_load(f, merge_duplicate_keys=True)
         return raw_config
 
 
@@ -724,7 +725,7 @@ def _write_sample_section(args, f, section_header, schema, as_comment=True, uwsg
         default = None if "default" not in value else value["default"]
         option = schema.get_app_option(key)
         option_value = OptionValue(key, default, option)
-        # support uWSGI "dumb yaml parser" (unbit/uwsgi#863)
+        # support uWSGI "dumb YAML parser" (unbit/uwsgi#863)
         key = option.get('key', key)
         _write_option(args, f, key, option_value, as_comment=as_comment, uwsgi_hack=uwsgi_hack)
 

lib/galaxy/dependencies/pipfiles/default/Pipfile

@@ -45,7 +45,7 @@ pyuwsgi = "*"
 pysam = "==0.15.2"
 bdbag = "*"
 bleach = "*"
-cwltool = "==1.0.20180721142728"
+cwltool = "==1.0.20191225192155"
 "bz2file" = {version = "*", markers = "python_version < '3.3'"}
 ipaddress = {version = "*", markers = "python_version < '3.3'"}
 isa-rwval = "*"

lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt

@@ -27,8 +27,8 @@ mock==3.0.5
 more-itertools==5.0.0
 nose==1.3.7
 nosehtml==0.4.5
-packaging==19.2
-pathlib2==2.3.2 ; python_version < '3.6'
+packaging==20.0
+pathlib2==2.3.5 ; python_version < '3.6'
 pathtools==0.1.2
 pluggy==0.13.1
 port-for==0.4

lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt

@@ -4,7 +4,6 @@ adal==1.2.2
 amqp==2.5.2
 appdirs==1.4.3
 attrs==19.3.0
-avro==1.8.1 ; python_version < '3'
 azure-common==1.1.14
 azure-cosmosdb-nspkg==2.0.2
 azure-cosmosdb-table==1.0.4
@@ -41,10 +40,11 @@ cliff==2.16.0
 cloudauthz==0.6.0
 cloudbridge==2.0.0
 cmd2==0.8.9
+coloredlogs==10.0
 configparser==4.0.2 ; python_version < '3.2'
 contextlib2==0.6.0.post1 ; python_version < '3.5'
 cryptography==2.8
-cwltool==1.0.20180721142728
+cwltool==1.0.20191225192155
 debtcollector==1.22.0
 decorator==4.4.1
 deprecated==1.2.7
@@ -66,7 +66,8 @@ google-auth-httplib2==0.0.3
 google-auth==1.7.1
 gxformat2==0.10.1
 h5py==2.10.0
-httplib2==0.14.0
+httplib2==0.15.0
+humanfriendly==4.18
 idna==2.8
 importlib-metadata==1.2.0 ; python_version < '3.8'
 ipaddress==1.0.23 ; python_version < '3.3'
@@ -111,13 +112,13 @@ oslo.i18n==3.25.0
 oslo.log==3.45.0
 oslo.serialization==2.29.2
 oslo.utils==3.42.0
-packaging==19.2
+packaging==20.0
 paramiko==2.7.1
 parsley==1.3
 paste==3.2.3
 pastedeploy==2.0.1
 pastescript==3.2.0
-pathlib2==2.3.2 ; python_version < '3.6'
+pathlib2==2.3.5 ; python_version < '3.6'
 pbr==5.4.4
 prettytable==0.7.2
 prov==1.5.1
@@ -162,10 +163,11 @@ rfc3986==1.3.2
 routes==2.4.1
 rsa==4.0
 ruamel.ordereddict==0.4.14 ; platform_python_implementation == 'CPython' and python_version <= '2.7'
-ruamel.yaml==0.15.100
+ruamel.yaml.clib==0.2.0 ; platform_python_implementation == 'CPython' and python_version < '3.8'
+ruamel.yaml==0.16.0
 s3transfer==0.2.1
 scandir==1.10.0 ; python_version < '3.5'
-schema-salad==2.7.20181126142424
+schema-salad==4.5.20191229160203
 setuptools-scm==3.3.3
 shellescape==3.4.1
 simplejson==3.17.0

lib/galaxy/tool_util/cwl/cwltool_deps.py

@@ -5,6 +5,9 @@
 functionality at runtime.
 """
 import re
+import warnings
+
+warnings.filterwarnings("ignore", message=r"[\n.]DEPRECATION: Python 2", module="cwltool")
 
 try:
     import requests
@@ -19,23 +22,44 @@
         process,
         pathmapper,
     )
-except (ImportError, SyntaxError):
-    # Drop SyntaxError once cwltool supports Python 3
+except ImportError:
     main = None
     workflow = None
     job = None
     process = None
     pathmapper = None
 
 try:
-    from cwltool.context import LoadingContext  # Introduced in cwltool 1.0.20180615183820
-except (ImportError, SyntaxError):
+    from cwltool.context import (
+        getdefault,
+        LoadingContext,
+        RuntimeContext,
+    )
+    from cwltool.job import relink_initialworkdir
+    from cwltool.stdfsaccess import StdFsAccess
+except ImportError:
+    getdefault = None
     LoadingContext = None
+    relink_initialworkdir = None
+    RuntimeContext = None
+    StdFsAccess = None
 
 try:
     from cwltool import load_tool
-except (ImportError, SyntaxError):
+    from cwltool.load_tool import (
+        default_loader,
+        resolve_and_validate_document,
+    )
+except ImportError:
+    default_loader = None
     load_tool = None
+    resolve_and_validate_document = None
+
+try:
+    from cwltool import command_line_tool
+    command_line_tool.ACCEPTLIST_RE = command_line_tool.ACCEPTLIST_EN_RELAXED_RE
+except ImportError:
+    command_line_tool = None
 
 try:
     import shellescape
@@ -44,18 +68,20 @@
 
 try:
     import schema_salad
-except (ImportError, SyntaxError):
-    # Drop SyntaxError once schema_salad supports Python 3
+    from schema_salad import (
+        ref_resolver,
+        sourceline,
+    )
+except ImportError:
     schema_salad = None
-
-try:
-    from schema_salad import ref_resolver
-except (ImportError, SyntaxError):
     ref_resolver = None
-
+    sourceline = None
 
 needs_shell_quoting = re.compile(r"""(^$|[\s|&;()<>\'"$@])""").search
 
+# if set to True, file format checking is not performed.
+beta_relaxed_fmt_check = True
+
 
 def ensure_cwltool_available():
     """Assert optional dependencies proxied via this module are available at runtime.
@@ -66,8 +92,8 @@ def ensure_cwltool_available():
         message = "This feature requires cwltool and dependencies to be available, they are not."
         if main is None:
             message += " cwltool is not unavailable."
-        elif load_tool is None:
-            message += " cwltool.load_tool is unavailable - cwltool version is too old."
+        elif resolve_and_validate_document is None:
+            message += " cwltool.load_tool.resolve_and_validate_document is unavailable - cwltool version is too old."
         if requests is None:
             message += " Library 'requests' unavailable."
         if shellescape is None:
@@ -78,15 +104,22 @@ def ensure_cwltool_available():
 
 
 __all__ = (
-    'main',
-    'ref_resolver',
+    'default_loader',
+    'ensure_cwltool_available',
+    'getdefault',
     'load_tool',
     'LoadingContext',
-    'workflow',
-    'process',
+    'main',
+    'needs_shell_quoting',
     'pathmapper',
-    'ensure_cwltool_available',
+    'process',
+    'ref_resolver',
+    'relink_initialworkdir',
+    'resolve_and_validate_document',
+    'RuntimeContext',
     'schema_salad',
     'shellescape',
-    'needs_shell_quoting',
+    'sourceline',
+    'StdFsAccess',
+    'workflow',
 )