Merge pull request #2354 from jmchilton/galaxy-lib-update

Update to latest galaxy-lib.

lib/galaxy/tools/cwl/cwltool_deps.py

@@ -1,5 +1,8 @@
-""" This module contains logic for dealing with cwltool as an optional
-dependency for Galaxy and/or applications which use Galaxy as a library.
+"""Logic for dealing with cwltool as an optional dependency.
+
+Use this as the import interface for cwltool and just call
+:func:`ensure_cwltool_available` before using any of the imported
+functionality at runtime.
 """
 
 try:
@@ -14,12 +17,18 @@ try:
         job,
         process,
     )
-except ImportError:
+except (ImportError, SyntaxError):
+    # Drop SyntaxError once cwltool supports Python 3
     main = None
     workflow = None
     job = None
     process = None
 
+try:
+    from cwltool import load_tool
+except (ImportError, SyntaxError):
+    load_tool = None
+
 try:
     import shellescape
 except ImportError:
@@ -27,7 +36,8 @@ except ImportError:
 
 try:
     import schema_salad
-except ImportError:
+except (ImportError, SyntaxError):
+    # Drop SyntaxError once schema_salad supports Python 3
     schema_salad = None
 
 import re
@@ -36,8 +46,16 @@ needs_shell_quoting = re.compile(r"""(^$|[\s|&;()<>\'"$@])""").search
 
 
 def ensure_cwltool_available():
+    """Assert optional dependencies proxied via this module are available at runtime.
+
+    Throw an ImportError with a description of the problem if they do not exist.
+    """
     if main is None or workflow is None or shellescape is None:
         message = "This feature requires cwltool and dependencies to be available, they are not."
+        if main is None:
+            message += " cwltool is not unavailable."
+        elif load_tool is None:
+            message += " cwltool.load_tool is unavailable - cwltool version is too old."
         if requests is None:
             message += " Library 'requests' unavailable."
         if shellescape is None:
@@ -49,6 +67,7 @@ def ensure_cwltool_available():
 
 __all__ = [
     'main',
+    'load_tool',
     'workflow',
     'process',
     'ensure_cwltool_available',

lib/galaxy/tools/cwl/parser.py

@@ -69,7 +69,6 @@ def load_job_proxy(job_directory):
 
 def to_cwl_tool_object(tool_path):
     proxy_class = None
-    cwl_tool = None
     cwl_tool = schema_loader.tool(path=tool_path)
     if isinstance(cwl_tool, int):
         raise Exception("Failed to load tool.")
@@ -88,10 +87,6 @@ def to_cwl_tool_object(tool_path):
     if "cwlVersion" not in raw_tool:
         raise Exception("File does not declare a CWL version, pre-draft 3 CWL tools are not supported.")
 
-    cwl_version = raw_tool["cwlVersion"]
-    if cwl_version != "https://w3id.org/cwl/cwl#draft-3":
-        raise Exception("Only draft 3 CWL tools are supported by Galaxy.")
-
     proxy = proxy_class(cwl_tool, tool_path)
     return proxy
 
@@ -154,14 +149,19 @@ class ToolProxy( object ):
     def description(self):
         """ Return description to tool. """
 
+    @abstractmethod
+    def label(self):
+        """ Return label for tool. """
+
 
 class CommandLineToolProxy(ToolProxy):
 
     def description(self):
-        # Feels like I should be getting some abstract namespaced thing
-        # not actually description, is this correct?
         return self._tool.tool.get('description')
 
+    def label(self):
+        return self._tool.tool.get('label')
+
     def input_instances(self):
         return self._find_inputs(self._tool.inputs_record_schema)
 

lib/galaxy/tools/cwl/schema.py

@@ -1,78 +1,72 @@
+"""Abstraction around cwltool and related libraries for loading a CWL artifact."""
 from collections import namedtuple
 import os
 
 from six.moves.urllib.parse import urldefrag
 
 from .cwltool_deps import (
-    process,
+    ensure_cwltool_available,
     schema_salad,
     workflow,
+    load_tool,
 )
 
-ProcessDefinition = namedtuple("ProcessObject", ["process_object", "metadata"])
+RawProcessReference = namedtuple("RawProcessReference", ["process_object", "uri"])
+ProcessDefinition = namedtuple("ProcessDefinition", ["process_object", "metadata", "document_loader", "avsc_names", "raw_process_reference"])
 
 
 class SchemaLoader(object):
 
     def __init__(self, strict=True):
         self._strict = strict
-        self._document_loader = None
-        self._avsc_names = None
+        self._raw_document_loader = None
 
-    def _lazy_init(self):
-        if self._document_loader is not None:
-            return
+    @property
+    def raw_document_loader(self):
+        if self._raw_document_loader is None:
+            ensure_cwltool_available()
+            self._raw_document_loader = schema_salad.ref_resolver.Loader({"cwl": "https://w3id.org/cwl/cwl#", "id": "@id"})
 
-        document_loader, avsc_names, _ = process.get_schema()
-        self._document_loader = document_loader
-        self._avsc_names = avsc_names
+        return self._raw_document_loader
 
-        if isinstance(avsc_names, Exception):
-            raise avsc_names
-
-    def raw_object(self, path):
-        self._lazy_init()
-        self._document_loader.idx.clear()
+    def raw_process_reference(self, path):
         uri = "file://" + os.path.abspath(path)
         fileuri, _ = urldefrag(uri)
-        return self._document_loader.fetch(fileuri)
+        return RawProcessReference(self.raw_document_loader.fetch(fileuri), uri)
 
-    def process_definition(self, raw_object):
-        self._lazy_init()
-        process_object, metadata = schema_salad.schema.load_and_validate(
-            self._document_loader,
-            self._avsc_names,
-            raw_object,
-            self._strict,
+    def process_definition(self, raw_reference):
+        document_loader, avsc_names, process_object, metadata, uri = load_tool.validate_document(
+            self.raw_document_loader,
+            raw_reference.process_object,
+            raw_reference.uri,
         )
-        return ProcessDefinition(process_object, metadata)
+        process_def = ProcessDefinition(
+            process_object,
+            metadata,
+            document_loader,
+            avsc_names,
+            raw_reference,
+        )
+        return process_def
 
     def tool(self, **kwds):
-        self._lazy_init()
         process_definition = kwds.get("process_definition", None)
         if process_definition is None:
-            raw_object = kwds.get("raw_object", None)
-            if raw_object is None:
-                raw_object = self.raw_object(kwds["path"])
-            process_definition = self.process_definition(raw_object)
+            raw_process_reference = kwds.get("raw_process_reference", None)
+            if raw_process_reference is None:
+                raw_process_reference = self.raw_process_reference(kwds["path"])
+            process_definition = self.process_definition(raw_process_reference)
 
         make_tool = kwds.get("make_tool", workflow.defaultMakeTool)
-        tool = make_tool(
+        tool = load_tool.make_tool(
+            process_definition.document_loader,
+            process_definition.avsc_names,
             process_definition.process_object,
-            strict=self._strict,
-            makeTool=make_tool,
-            loader=self._document_loader,
-            avsc_names=self._avsc_names,
+            process_definition.metadata,
+            process_definition.raw_process_reference.uri,
+            make_tool,
+            {"strict": self._strict},
         )
-        if process_definition.metadata:
-            metadata = process_definition.metadata
-        else:
-            metadata = {
-                "$namespaces": tool.tool.get("$namespaces", {}),
-                "$schemas": tool.tool.get("$schemas", [])
-            }
-
-        tool.metadata = metadata
         return tool
 
 schema_loader = SchemaLoader()

lib/galaxy/tools/lint.py

@@ -31,6 +31,11 @@ def lint_tool_source_with(lint_context, tool_source, extra_modules=[]):
     linter_modules = submodules.submodules(galaxy.tools.linters)
     linter_modules.extend(extra_modules)
     for module in linter_modules:
+        tool_type = tool_source.parse_tool_type() or "default"
+        lint_tool_types = getattr(module, "lint_tool_types", ["default"])
+        if not ("*" in lint_tool_types or tool_type in lint_tool_types):
+            continue
+
         for (name, value) in inspect.getmembers(module):
             if callable(value) and name.startswith("lint_"):
                 # Look at the first argument to the linter to decide

lib/galaxy/tools/linters/general.py

@@ -10,6 +10,8 @@ VALID_NAME_MSG = "Tool defines a name [%s]."
 ERROR_ID_MSG = "Tool does not define an id attribute."
 VALID_ID_MSG = "Tool defines an id [%s]."
 
+lint_tool_types = ["*"]
+
 
 def lint_general(tool_source, lint_ctx):
     """Check tool version, name, and id."""

lib/galaxy/tools/loader_directory.py

@@ -219,7 +219,7 @@ def looks_like_a_cwl_artifact(path, classes=None):
 
 def looks_like_a_tool_cwl(path):
     """Quick check to see if a file looks like it may be a CWL tool."""
-    return looks_like_a_cwl_artifact(path, classes=["CommandLineTool"])
+    return looks_like_a_cwl_artifact(path, classes=["CommandLineTool", "ExpressionTool"])
 
 
 def _find_tool_files(path, recursive, enable_beta_formats):

lib/galaxy/tools/parser/cwl.py

@@ -23,22 +23,23 @@ class CwlToolSource(ToolSource):
     def __init__(self, tool_file):
         self._cwl_tool_file = tool_file
         self._id, _ = os.path.splitext(os.path.basename(tool_file))
-        self._tool_proxy = tool_proxy(tool_file)
+        self._tool_proxy = None
         self._source_path = tool_file
 
     @property
     def tool_proxy(self):
+        if self._tool_proxy is None:
+            self._tool_proxy = tool_proxy(self._source_path)
         return self._tool_proxy
 
     def parse_tool_type(self):
         return 'cwl'
 
     def parse_id(self):
-        log.warn("TOOL ID is %s" % self._id)
         return self._id
 
     def parse_name(self):
-        return self._id
+        return self.tool_proxy.label() or self.parse_id()
 
     def parse_command(self):
         return "$__cwl_command"
@@ -60,7 +61,7 @@ class CwlToolSource(ToolSource):
         return environment_variables
 
     def parse_help(self):
-        return ""
+        return self.tool_proxy.description() or ""
 
     def parse_sanitize(self):
         return False
@@ -90,14 +91,14 @@ class CwlToolSource(ToolSource):
         return "0.0.1"
 
     def parse_description(self):
-        return self._tool_proxy.description() or ""
+        return ""
 
     def parse_input_pages(self):
-        page_source = CwlPageSource(self._tool_proxy)
+        page_source = CwlPageSource(self.tool_proxy)
         return PagesSource([page_source])
 
     def parse_outputs(self, tool):
-        output_instances = self._tool_proxy.output_instances()
+        output_instances = self.tool_proxy.output_instances()
         outputs = odict()
         output_defs = []
         for output_instance in output_instances:
@@ -130,7 +131,7 @@ class CwlToolSource(ToolSource):
 
     def parse_requirements_and_containers(self):
         containers = []
-        docker_identifier = self._tool_proxy.docker_identifier()
+        docker_identifier = self.tool_proxy.docker_identifier()
         if docker_identifier:
             containers.append({"type": "docker",
                                "identifier": docker_identifier})