diff --git a/lib/galaxy/tools/cwl/cwltool_deps.py b/lib/galaxy/tools/cwl/cwltool_deps.py index ff0f5f00c43..376fd30f5ca 100644 --- a/lib/galaxy/tools/cwl/cwltool_deps.py +++ b/lib/galaxy/tools/cwl/cwltool_deps.py @@ -1,5 +1,8 @@ -""" This module contains logic for dealing with cwltool as an optional -dependency for Galaxy and/or applications which use Galaxy as a library. +"""Logic for dealing with cwltool as an optional dependency. + +Use this as the import interface for cwltool and just call +:func:`ensure_cwltool_available` before using any of the imported +functionality at runtime. """ try: @@ -14,12 +17,18 @@ try: job, process, ) -except ImportError: +except (ImportError, SyntaxError): + # Drop SyntaxError once cwltool supports Python 3 main = None workflow = None job = None process = None +try: + from cwltool import load_tool +except (ImportError, SyntaxError): + load_tool = None + try: import shellescape except ImportError: @@ -27,7 +36,8 @@ except ImportError: try: import schema_salad -except ImportError: +except (ImportError, SyntaxError): + # Drop SyntaxError once schema_salad supports Python 3 schema_salad = None import re @@ -36,8 +46,16 @@ needs_shell_quoting = re.compile(r"""(^$|[\s|&;()<>\'"$@])""").search def ensure_cwltool_available(): + """Assert optional dependencies proxied via this module are available at runtime. + + Throw an ImportError with a description of the problem if they do not exist. + """ if main is None or workflow is None or shellescape is None: message = "This feature requires cwltool and dependencies to be available, they are not." + if main is None: + message += " cwltool is not unavailable." + elif load_tool is None: + message += " cwltool.load_tool is unavailable - cwltool version is too old." if requests is None: message += " Library 'requests' unavailable." if shellescape is None: @@ -49,6 +67,7 @@ def ensure_cwltool_available(): __all__ = [ 'main', + 'load_tool', 'workflow', 'process', 'ensure_cwltool_available', diff --git a/lib/galaxy/tools/cwl/parser.py b/lib/galaxy/tools/cwl/parser.py index a2a06c60563..bbcc4d9f680 100644 --- a/lib/galaxy/tools/cwl/parser.py +++ b/lib/galaxy/tools/cwl/parser.py @@ -69,7 +69,6 @@ def load_job_proxy(job_directory): def to_cwl_tool_object(tool_path): proxy_class = None - cwl_tool = None cwl_tool = schema_loader.tool(path=tool_path) if isinstance(cwl_tool, int): raise Exception("Failed to load tool.") @@ -88,10 +87,6 @@ def to_cwl_tool_object(tool_path): if "cwlVersion" not in raw_tool: raise Exception("File does not declare a CWL version, pre-draft 3 CWL tools are not supported.") - cwl_version = raw_tool["cwlVersion"] - if cwl_version != "https://w3id.org/cwl/cwl#draft-3": - raise Exception("Only draft 3 CWL tools are supported by Galaxy.") - proxy = proxy_class(cwl_tool, tool_path) return proxy @@ -154,14 +149,19 @@ class ToolProxy( object ): def description(self): """ Return description to tool. """ + @abstractmethod + def label(self): + """ Return label for tool. """ + class CommandLineToolProxy(ToolProxy): def description(self): - # Feels like I should be getting some abstract namespaced thing - # not actually description, is this correct? return self._tool.tool.get('description') + def label(self): + return self._tool.tool.get('label') + def input_instances(self): return self._find_inputs(self._tool.inputs_record_schema) diff --git a/lib/galaxy/tools/cwl/schema.py b/lib/galaxy/tools/cwl/schema.py index 9f08f0d2329..4ecbd0f7a2a 100644 --- a/lib/galaxy/tools/cwl/schema.py +++ b/lib/galaxy/tools/cwl/schema.py @@ -1,78 +1,72 @@ +"""Abstraction around cwltool and related libraries for loading a CWL artifact.""" from collections import namedtuple import os from six.moves.urllib.parse import urldefrag from .cwltool_deps import ( - process, + ensure_cwltool_available, schema_salad, workflow, + load_tool, ) -ProcessDefinition = namedtuple("ProcessObject", ["process_object", "metadata"]) +RawProcessReference = namedtuple("RawProcessReference", ["process_object", "uri"]) +ProcessDefinition = namedtuple("ProcessDefinition", ["process_object", "metadata", "document_loader", "avsc_names", "raw_process_reference"]) class SchemaLoader(object): def __init__(self, strict=True): self._strict = strict - self._document_loader = None - self._avsc_names = None + self._raw_document_loader = None - def _lazy_init(self): - if self._document_loader is not None: - return + @property + def raw_document_loader(self): + if self._raw_document_loader is None: + ensure_cwltool_available() + self._raw_document_loader = schema_salad.ref_resolver.Loader({"cwl": "https://w3id.org/cwl/cwl#", "id": "@id"}) - document_loader, avsc_names, _ = process.get_schema() - self._document_loader = document_loader - self._avsc_names = avsc_names + return self._raw_document_loader - if isinstance(avsc_names, Exception): - raise avsc_names - - def raw_object(self, path): - self._lazy_init() - self._document_loader.idx.clear() + def raw_process_reference(self, path): uri = "file://" + os.path.abspath(path) fileuri, _ = urldefrag(uri) - return self._document_loader.fetch(fileuri) + return RawProcessReference(self.raw_document_loader.fetch(fileuri), uri) - def process_definition(self, raw_object): - self._lazy_init() - process_object, metadata = schema_salad.schema.load_and_validate( - self._document_loader, - self._avsc_names, - raw_object, - self._strict, + def process_definition(self, raw_reference): + document_loader, avsc_names, process_object, metadata, uri = load_tool.validate_document( + self.raw_document_loader, + raw_reference.process_object, + raw_reference.uri, ) - return ProcessDefinition(process_object, metadata) + process_def = ProcessDefinition( + process_object, + metadata, + document_loader, + avsc_names, + raw_reference, + ) + return process_def def tool(self, **kwds): - self._lazy_init() process_definition = kwds.get("process_definition", None) if process_definition is None: - raw_object = kwds.get("raw_object", None) - if raw_object is None: - raw_object = self.raw_object(kwds["path"]) - process_definition = self.process_definition(raw_object) + raw_process_reference = kwds.get("raw_process_reference", None) + if raw_process_reference is None: + raw_process_reference = self.raw_process_reference(kwds["path"]) + process_definition = self.process_definition(raw_process_reference) make_tool = kwds.get("make_tool", workflow.defaultMakeTool) - tool = make_tool( + tool = load_tool.make_tool( + process_definition.document_loader, + process_definition.avsc_names, process_definition.process_object, - strict=self._strict, - makeTool=make_tool, - loader=self._document_loader, - avsc_names=self._avsc_names, + process_definition.metadata, + process_definition.raw_process_reference.uri, + make_tool, + {"strict": self._strict}, ) - if process_definition.metadata: - metadata = process_definition.metadata - else: - metadata = { - "$namespaces": tool.tool.get("$namespaces", {}), - "$schemas": tool.tool.get("$schemas", []) - } - - tool.metadata = metadata return tool schema_loader = SchemaLoader() diff --git a/lib/galaxy/tools/lint.py b/lib/galaxy/tools/lint.py index 107fcb4c243..7f99d702560 100644 --- a/lib/galaxy/tools/lint.py +++ b/lib/galaxy/tools/lint.py @@ -31,6 +31,11 @@ def lint_tool_source_with(lint_context, tool_source, extra_modules=[]): linter_modules = submodules.submodules(galaxy.tools.linters) linter_modules.extend(extra_modules) for module in linter_modules: + tool_type = tool_source.parse_tool_type() or "default" + lint_tool_types = getattr(module, "lint_tool_types", ["default"]) + if not ("*" in lint_tool_types or tool_type in lint_tool_types): + continue + for (name, value) in inspect.getmembers(module): if callable(value) and name.startswith("lint_"): # Look at the first argument to the linter to decide diff --git a/lib/galaxy/tools/linters/general.py b/lib/galaxy/tools/linters/general.py index 8b6b077755a..16dc43a8532 100644 --- a/lib/galaxy/tools/linters/general.py +++ b/lib/galaxy/tools/linters/general.py @@ -10,6 +10,8 @@ VALID_NAME_MSG = "Tool defines a name [%s]." ERROR_ID_MSG = "Tool does not define an id attribute." VALID_ID_MSG = "Tool defines an id [%s]." +lint_tool_types = ["*"] + def lint_general(tool_source, lint_ctx): """Check tool version, name, and id.""" diff --git a/lib/galaxy/tools/loader_directory.py b/lib/galaxy/tools/loader_directory.py index dad0643fb0c..9551c54c96a 100644 --- a/lib/galaxy/tools/loader_directory.py +++ b/lib/galaxy/tools/loader_directory.py @@ -219,7 +219,7 @@ def looks_like_a_cwl_artifact(path, classes=None): def looks_like_a_tool_cwl(path): """Quick check to see if a file looks like it may be a CWL tool.""" - return looks_like_a_cwl_artifact(path, classes=["CommandLineTool"]) + return looks_like_a_cwl_artifact(path, classes=["CommandLineTool", "ExpressionTool"]) def _find_tool_files(path, recursive, enable_beta_formats): diff --git a/lib/galaxy/tools/parser/cwl.py b/lib/galaxy/tools/parser/cwl.py index 3f0a7bd526e..b9a4731e712 100644 --- a/lib/galaxy/tools/parser/cwl.py +++ b/lib/galaxy/tools/parser/cwl.py @@ -23,22 +23,23 @@ class CwlToolSource(ToolSource): def __init__(self, tool_file): self._cwl_tool_file = tool_file self._id, _ = os.path.splitext(os.path.basename(tool_file)) - self._tool_proxy = tool_proxy(tool_file) + self._tool_proxy = None self._source_path = tool_file @property def tool_proxy(self): + if self._tool_proxy is None: + self._tool_proxy = tool_proxy(self._source_path) return self._tool_proxy def parse_tool_type(self): return 'cwl' def parse_id(self): - log.warn("TOOL ID is %s" % self._id) return self._id def parse_name(self): - return self._id + return self.tool_proxy.label() or self.parse_id() def parse_command(self): return "$__cwl_command" @@ -60,7 +61,7 @@ class CwlToolSource(ToolSource): return environment_variables def parse_help(self): - return "" + return self.tool_proxy.description() or "" def parse_sanitize(self): return False @@ -90,14 +91,14 @@ class CwlToolSource(ToolSource): return "0.0.1" def parse_description(self): - return self._tool_proxy.description() or "" + return "" def parse_input_pages(self): - page_source = CwlPageSource(self._tool_proxy) + page_source = CwlPageSource(self.tool_proxy) return PagesSource([page_source]) def parse_outputs(self, tool): - output_instances = self._tool_proxy.output_instances() + output_instances = self.tool_proxy.output_instances() outputs = odict() output_defs = [] for output_instance in output_instances: @@ -130,7 +131,7 @@ class CwlToolSource(ToolSource): def parse_requirements_and_containers(self): containers = [] - docker_identifier = self._tool_proxy.docker_identifier() + docker_identifier = self.tool_proxy.docker_identifier() if docker_identifier: containers.append({"type": "docker", "identifier": docker_identifier})