Merge pull request #3742 from jmchilton/galaxy-lib-20170311

Mirror latest galaxy-lib changes for Conda and CWL improvements.

lib/galaxy/tools/__init__.py

@@ -42,6 +42,7 @@ from galaxy.tools.deps import (
     CachedDependencyManager,
     views
 )
+from galaxy.tools.fetcher import ToolLocationFetcher
 from galaxy.tools.parameters import (
     check_param,
     params_from_strings,
@@ -181,6 +182,7 @@ class ToolBox( BaseGalaxyToolBox ):
 
     def __init__( self, config_filenames, tool_root_dir, app, tool_conf_watcher=None ):
         self._reload_count = 0
+        self.tool_location_fetcher = ToolLocationFetcher()
         super( ToolBox, self ).__init__(
             config_filenames=config_filenames,
             tool_root_dir=tool_root_dir,
@@ -216,7 +218,11 @@ class ToolBox( BaseGalaxyToolBox ):
 
     def create_tool( self, config_file, repository_id=None, guid=None, **kwds ):
         try:
-            tool_source = get_tool_source( config_file, enable_beta_formats=getattr( self.app.config, "enable_beta_tool_formats", False ) )
+            tool_source = get_tool_source(
+                config_file,
+                enable_beta_formats=getattr( self.app.config, "enable_beta_tool_formats", False ),
+                tool_location_fetcher=self.tool_location_fetcher,
+            )
         except Exception as e:
             # capture and log parsing errors
             global_tool_errors.add_error(config_file, "Tool XML parsing", e)

lib/galaxy/tools/cwl/cwltool_deps.py

@@ -17,6 +17,7 @@ try:
         workflow,
         job,
         process,
+        pathmapper,
     )
 except (ImportError, SyntaxError):
     # Drop SyntaxError once cwltool supports Python 3
@@ -24,6 +25,7 @@ except (ImportError, SyntaxError):
     workflow = None
     job = None
     process = None
+    pathmapper = None
 
 try:
     from cwltool import load_tool
@@ -41,6 +43,12 @@ except (ImportError, SyntaxError):
     # Drop SyntaxError once schema_salad supports Python 3
     schema_salad = None
 
+try:
+    from schema_salad import ref_resolver
+except (ImportError, SyntaxError):
+    ref_resolver = None
+
+
 needs_shell_quoting = re.compile(r"""(^$|[\s|&;()<>\'"$@])""").search
 
 
@@ -66,9 +74,11 @@ def ensure_cwltool_available():
 
 __all__ = (
     'main',
+    'ref_resolver',
     'load_tool',
     'workflow',
     'process',
+    'pathmapper',
     'ensure_cwltool_available',
     'schema_salad',
     'shellescape',

lib/galaxy/tools/cwl/parser.py

@@ -18,11 +18,10 @@ from galaxy.util.odict import odict
 
 from .cwltool_deps import (
     ensure_cwltool_available,
-    main,
-    workflow,
+    process,
 )
 
-from .schema import schema_loader
+from .schema import non_strict_schema_loader, schema_loader
 
 log = logging.getLogger(__name__)
 
@@ -42,36 +41,36 @@ SUPPORTED_WORKFLOW_REQUIREMENTS = SUPPORTED_TOOL_REQUIREMENTS + [
 ]
 
 
-def tool_proxy(tool_path):
+def tool_proxy(tool_path, strict_cwl_validation=True):
     """ Provide a proxy object to cwltool data structures to just
     grab relevant data.
     """
     ensure_cwltool_available()
-    tool = to_cwl_tool_object(tool_path)
+    tool = to_cwl_tool_object(tool_path, strict_cwl_validation=strict_cwl_validation)
     return tool
 
 
-def workflow_proxy(workflow_path):
+def workflow_proxy(workflow_path, strict_cwl_validation=True):
     ensure_cwltool_available()
-    workflow = to_cwl_workflow_object(workflow_path)
+    workflow = to_cwl_workflow_object(workflow_path, strict_cwl_validation=strict_cwl_validation)
     return workflow
 
 
-def load_job_proxy(job_directory):
+def load_job_proxy(job_directory, strict_cwl_validation=True):
     ensure_cwltool_available()
     job_objects_path = os.path.join(job_directory, JOB_JSON_FILE)
     job_objects = json.load(open(job_objects_path, "r"))
     tool_path = job_objects["tool_path"]
     job_inputs = job_objects["job_inputs"]
     output_dict = job_objects["output_dict"]
-    cwl_tool = tool_proxy(tool_path)
+    cwl_tool = tool_proxy(tool_path, strict_cwl_validation=strict_cwl_validation)
     cwl_job = cwl_tool.job_proxy(job_inputs, output_dict, job_directory=job_directory)
     return cwl_job
 
 
-def to_cwl_tool_object(tool_path):
+def to_cwl_tool_object(tool_path, strict_cwl_validation=True):
     proxy_class = None
-    cwl_tool = schema_loader.tool(path=tool_path)
+    cwl_tool = _schema_loader(strict_cwl_validation).tool(path=tool_path)
     if isinstance(cwl_tool, int):
         raise Exception("Failed to load tool.")
 
@@ -93,10 +92,9 @@ def to_cwl_tool_object(tool_path):
     return proxy
 
 
-def to_cwl_workflow_object(workflow_path):
+def to_cwl_workflow_object(workflow_path, strict_cwl_validation=None):
     proxy_class = WorkflowProxy
-    make_tool = workflow.defaultMakeTool
-    cwl_workflow = main.load_tool(workflow_path, False, False, make_tool, False)
+    cwl_workflow = _schema_loader(strict_cwl_validation).tool(path=workflow_path)
     raw_workflow = cwl_workflow.tool
     check_requirements(raw_workflow, tool=False)
 
@@ -104,6 +102,11 @@ def to_cwl_workflow_object(workflow_path):
     return proxy
 
 
+def _schema_loader(strict_cwl_validation):
+    target_schema_loader = schema_loader if strict_cwl_validation else non_strict_schema_loader
+    return target_schema_loader
+
+
 def check_requirements(rec, tool=True):
     if isinstance(rec, dict):
         if "requirements" in rec:
@@ -237,14 +240,21 @@ class JobProxy(object):
 
     def _ensure_cwl_job_initialized(self):
         if self._cwl_job is None:
+
             self._cwl_job = next(self._tool_proxy._tool.job(
                 self._input_dict,
                 self._output_callback,
                 basedir=self._job_directory,
+                select_resources=self._select_resources,
                 use_container=False
             ))
             self._is_command_line_job = hasattr(self._cwl_job, "command_line")
 
+    def _select_resources(self, request):
+        new_request = request.copy()
+        new_request["cores"] = "$GALAXY_SLOTS"
+        return new_request
+
     @property
     def command_line(self):
         if self.is_command_line_job:
@@ -324,6 +334,12 @@ class JobProxy(object):
             safe_makedirs(secondary_files_dir)
         return secondary_files_dir
 
+    def stage_files(self):
+        cwl_job = self.cwl_job()
+        if hasattr(cwl_job, "pathmapper"):
+            process.stageFiles(self.cwl_job().pathmapper, os.symlink, ignoreWritable=True)
+        # else: expression tools do not have a path mapper.
+
     @staticmethod
     def _job_file(job_directory):
         return os.path.join(job_directory, JOB_JSON_FILE)

lib/galaxy/tools/cwl/representation.py

@@ -19,6 +19,7 @@ GALAXY_TO_CWL_TYPES = {
     'float': 'float',
     'data': 'File',
     'boolean': 'boolean',
+    'text': 'text'
 }
 
 
@@ -56,7 +57,8 @@ def to_cwl_job(tool, param_dict, local_working_directory):
                     os.symlink(secondary_file_path, new_input_path + secondary_file_name)
                 path = new_input_path
 
-            return {"path": path, "class": "File"}
+            return {"location": path,
+                    "class": "File"}
         elif cwl_type == "integer":
             return int(str(param_dict_value))
         elif cwl_type == "long":
@@ -67,7 +69,7 @@ def to_cwl_job(tool, param_dict, local_working_directory):
             return float(str(param_dict_value))
         elif cwl_type == "boolean":
             return string_as_bool(param_dict_value)
-        elif cwl_type == "string":
+        elif cwl_type == "text":
             return str(param_dict_value)
         elif cwl_type == "json":
             raw_value = param_dict_value.value
@@ -94,9 +96,6 @@ def to_cwl_job(tool, param_dict, local_working_directory):
         else:
             input_json[input_name] = simple_value(input, param_dict[input_name])
 
-    input_json["allocatedResources"] = {
-        "cpu": "$GALAXY_SLOTS",
-    }
     return input_json
 
 

lib/galaxy/tools/cwl/runtime_actions.py

@@ -2,6 +2,7 @@ import json
 import os
 import shutil
 
+from .cwltool_deps import ref_resolver
 from .parser import (
     JOB_JSON_FILE,
     load_job_proxy,
@@ -17,13 +18,16 @@ def handle_outputs(job_directory=None):
     if not os.path.exists(cwl_job_file):
         # Not a CWL job, just continue
         return
-    job_proxy = load_job_proxy(job_directory)
+    # So we only need to do strict validation when the tool was loaded,
+    # no reason to do it again during job execution - so this shortcut
+    # allows us to not need Galaxy's full configuration on job nodes.
+    job_proxy = load_job_proxy(job_directory, strict_cwl_validation=False)
     tool_working_directory = os.path.join(job_directory, "working")
     outputs = job_proxy.collect_outputs(tool_working_directory)
     for output_name, output in outputs.items():
-        target_path = job_proxy.output_path( output_name )
-        if isinstance(output, dict) and "path" in output:
-            output_path = output["path"]
+        target_path = job_proxy.output_path(output_name)
+        if isinstance(output, dict) and "location" in output:
+            output_path = ref_resolver.uri_file_path(output["location"])
             if output["class"] != "File":
                 open("galaxy.json", "w").write(json.dump({
                     "dataset_id": job_proxy.output_id(output_name),
@@ -33,7 +37,7 @@ def handle_outputs(job_directory=None):
             shutil.move(output_path, target_path)
             for secondary_file in output.get("secondaryFiles", []):
                 # TODO: handle nested files...
-                secondary_file_path = secondary_file["path"]
+                secondary_file_path = ref_resolver.uri_file_path(secondary_file["location"])
                 assert secondary_file_path.startswith(output_path)
                 secondary_file_name = secondary_file_path[len(output_path):]
                 secondary_files_dir = job_proxy.output_secondary_files_dir(

lib/galaxy/tools/cwl/schema.py

@@ -24,11 +24,8 @@ class SchemaLoader(object):
 
     @property
     def raw_document_loader(self):
-        if self._raw_document_loader is None:
-            ensure_cwltool_available()
-            self._raw_document_loader = schema_salad.ref_resolver.Loader({"cwl": "https://w3id.org/cwl/cwl#", "id": "@id"})
-
-        return self._raw_document_loader
+        ensure_cwltool_available()
+        return schema_salad.ref_resolver.Loader({"cwl": "https://w3id.org/cwl/cwl#", "id": "@id"})
 
     def raw_process_reference(self, path):
         uri = "file://" + os.path.abspath(path)
@@ -71,3 +68,4 @@ class SchemaLoader(object):
 
 
 schema_loader = SchemaLoader()
+non_strict_schema_loader = SchemaLoader(strict=False)

lib/galaxy/tools/deps/conda_util.py

@@ -30,6 +30,10 @@ VERSIONED_ENV_DIR_NAME = re.compile(r"__(.*)@(.*)")
 UNVERSIONED_ENV_DIR_NAME = re.compile(r"__(.*)@_uv_")
 USE_PATH_EXEC_DEFAULT = False
 CONDA_VERSION = "4.2.13"
+# 2.1.5 is incompatible with 4.2 Conda because of
+# https://github.com/conda/conda-build/pull/1766
+CONDA_BUILD_VERSION = "2.1.4"
+USE_LOCAL_DEFAULT = False
 
 
 def conda_link():
@@ -45,7 +49,16 @@ def find_conda_prefix(conda_prefix=None):
     for Miniconda installs.
     """
     if conda_prefix is None:
-        return os.path.join(os.path.expanduser("~"), "miniconda2")
+        home = os.path.expanduser("~")
+        miniconda_2_dest = os.path.join(home, "miniconda2")
+        miniconda_3_dest = os.path.join(home, "miniconda3")
+        # Prefer miniconda3 install if both available
+        if os.path.exists(miniconda_3_dest):
+            return miniconda_3_dest
+        elif os.path.exists(miniconda_2_dest):
+            return miniconda_2_dest
+        else:
+            return miniconda_3_dest
     return conda_prefix
 
 
@@ -54,7 +67,8 @@ class CondaContext(installable.InstallableContext):
 
     def __init__(self, conda_prefix=None, conda_exec=None,
                  shell_exec=None, debug=False, ensure_channels='',
-                 condarc_override=None, use_path_exec=USE_PATH_EXEC_DEFAULT, copy_dependencies=False):
+                 condarc_override=None, use_path_exec=USE_PATH_EXEC_DEFAULT,
+                 copy_dependencies=False, use_local=USE_LOCAL_DEFAULT):
         self.condarc_override = condarc_override
         if not conda_exec and use_path_exec:
             conda_exec = commands.which("conda")
@@ -84,18 +98,27 @@ class CondaContext(installable.InstallableContext):
         self.ensured_channels = False
         self._conda_version = None
         self._miniconda_version = None
+        self._conda_build_available = None
+        self.use_local = use_local
 
     @property
     def conda_version(self):
         if self._conda_version is None:
-            self._guess_conda_version()
+            self._guess_conda_properties()
         return self._conda_version
 
-    def _guess_conda_version(self):
+    @property
+    def conda_build_available(self):
+        if self._conda_build_available is None:
+            self._guess_conda_properties()
+        return self._conda_build_available
+
+    def _guess_conda_properties(self):
         conda_meta_path = self._conda_meta_path
         # Perhaps we should call "conda info --json" and parse it but for now we are going
         # to assume the default.
         conda_version = LooseVersion(CONDA_VERSION)
+        conda_build_available = False
         miniconda_version = "3"
 
         if os.path.exists(conda_meta_path):
@@ -110,9 +133,12 @@ class CondaContext(installable.InstallableContext):
                     conda_version = LooseVersion(version)
                 if package == "python" and version.startswith("2"):
                     miniconda_version = "2"
+                if package == "conda-build":
+                    conda_build_available = True
 
         self._conda_version = conda_version
         self._miniconda_version = miniconda_version
+        self._conda_build_available = conda_build_available
 
     @property
     def _conda_meta_path(self):
@@ -135,6 +161,14 @@ class CondaContext(installable.InstallableContext):
             if changed:
                 self.save_condarc(conda_conf)
 
+    def ensure_conda_build_installed_if_needed(self):
+        if self.use_local and not self.conda_build_available:
+            conda_targets = [CondaTarget("conda-build", version=CONDA_BUILD_VERSION)]
+            # Cannot use --use-local during installation fo conda-build.
+            return install_conda_targets(conda_targets, env_name=None, conda_context=self, allow_local=False)
+        else:
+            return 0
+
     def conda_info(self):
         if self.conda_exec is not None:
             info_out = commands.execute([self.conda_exec, "info", "--json"])
@@ -229,10 +263,12 @@ class CondaContext(installable.InstallableContext):
         finally:
             shutil.rmtree(env['HOME'], ignore_errors=True)
 
-    def exec_create(self, args):
+    def exec_create(self, args, allow_local=True):
         create_base_args = [
             "-y"
         ]
+        if allow_local and self.use_local:
+            create_base_args.extend(["--use-local"])
         create_base_args.extend(args)
         return self.exec_command("create", create_base_args)
 
@@ -246,10 +282,12 @@ class CondaContext(installable.InstallableContext):
         remove_base_args.extend(args)
         return self.exec_command("env", remove_base_args)
 
-    def exec_install(self, args):
+    def exec_install(self, args, allow_local=True):
         install_base_args = [
             "-y"
         ]
+        if allow_local and self.use_local:
+            install_base_args.extend(["--use-local"])
         install_base_args.extend(args)
         return self.exec_command("install", install_base_args)
 
@@ -390,13 +428,18 @@ def hash_conda_packages(conda_packages, conda_target=None):
 
 # shell makes sense for planemo, in Galaxy this should just execute
 # these commands as Python
-def install_conda(conda_context=None):
+def install_conda(conda_context=None, force_conda_build=False):
     conda_context = _ensure_conda_context(conda_context)
     f, script_path = tempfile.mkstemp(suffix=".sh", prefix="conda_install")
     os.close(f)
     download_cmd = " ".join(commands.download_command(conda_link(), to=script_path, quote_url=True))
     install_cmd = "bash '%s' -b -p '%s'" % (script_path, conda_context.conda_prefix)
-    fix_version_cmd = "%s install -y -q conda=%s " % (os.path.join(conda_context.conda_prefix, 'bin/conda'), CONDA_VERSION)
+    package_targets = [
+        "conda=%s" % CONDA_VERSION,
+    ]
+    if force_conda_build or conda_context.use_local:
+        package_targets.append("conda-build=%s" % CONDA_BUILD_VERSION)
+    fix_version_cmd = "%s install -y -q %s " % (os.path.join(conda_context.conda_prefix, 'bin/conda'), " ".join(package_targets))
     full_command = "%s && %s && %s" % (download_cmd, install_cmd, fix_version_cmd)
     try:
         log.info("Installing Conda, this may take several minutes.")
@@ -406,7 +449,7 @@ def install_conda(conda_context=None):
             os.remove(script_path)
 
 
-def install_conda_targets(conda_targets, env_name=None, conda_context=None):
+def install_conda_targets(conda_targets, env_name=None, conda_context=None, allow_local=True):
     conda_context = _ensure_conda_context(conda_context)
     conda_context.ensure_channels_configured()
     if env_name is not None:
@@ -415,9 +458,9 @@ def install_conda_targets(conda_targets, env_name=None, conda_context=None):
         ]
         for conda_target in conda_targets:
             create_args.append(conda_target.package_specifier)
-        return conda_context.exec_create(create_args)
+        return conda_context.exec_create(create_args, allow_local=allow_local)
     else:
-        return conda_context.exec_install([t.package_specifier for t in conda_targets])
+        return conda_context.exec_install([t.package_specifier for t in conda_targets], allow_local=allow_local)
 
 
 def install_conda_target(conda_target, conda_context=None, skip_environment=False):

lib/galaxy/tools/deps/mulled/invfile.lua

@@ -40,19 +40,49 @@ for i = 1, #test_binds_table do
     table.insert(test_bind_args, test_binds_table[i])
 end
 
+local conda_image = VAR.CONDA_IMAGE
+if conda_image == '' then
+    conda_image = 'continuumio/miniconda:latest'
+end
+
+local destination_base_image = VAR.DEST_BASE_IMAGE
+if destination_base_image == '' then
+    destination_base_image = 'bgruening/busybox-bash:0.1'
+end
+
+local verbose = VAR.VERBOSE
+if verbose == '' then
+    verbose = '--quiet'
+else
+    verbose = '--verbose'
+end
+
+local preinstall = VAR.PREINSTALL
+if preinstall ~= '' then
+    preinstall = preinstall .. ' && '
+end
+
+local postinstall = VAR.POSTINSTALL
+if postinstall ~= '' then
+    postinstall = '&&' .. postinstall
+end
+
 inv.task('build')
-    .using('continuumio/miniconda:latest')
+    .using(conda_image)
         .withHostConfig({binds = {"build:/data"}})
         .run('rm', '-rf', '/data/dist')
-    .using('continuumio/miniconda:latest')
+    .using(conda_image)
         .withHostConfig({binds = bind_args})
-        .run('/bin/sh', '-c', 'conda install '
+        .run('/bin/sh', '-c', preinstall
+            .. 'conda install '
             .. channel_args .. ' '
             .. target_args
-            .. ' -p /usr/local --copy --yes --quiet')
+            .. ' -p /usr/local --copy --yes '
+            .. verbose
+            .. postinstall)
     .wrap('build/dist')
         .at('/usr/local')
-        .inImage('bgruening/busybox-bash:0.1')
+        .inImage(destination_base_image)
         .as(repo)
 
 if VAR.TEST_BINDS == '' then

lib/galaxy/tools/deps/mulled/mulled_build.py

@@ -116,7 +116,7 @@ def mull_targets(
     command="build", channels=DEFAULT_CHANNELS, namespace="mulled",
     test='true', test_files=None, image_build=None, name_override=None,
     repository_template=DEFAULT_REPOSITORY_TEMPLATE, dry_run=False,
-    binds=DEFAULT_BINDS
+    conda_version=None, verbose=False, binds=DEFAULT_BINDS
 ):
     targets = list(targets)
     if involucro_context is None:
@@ -143,8 +143,13 @@ def mull_targets(
         '-set', "TARGETS='%s'" % target_str,
         '-set', "REPO='%s'" % repo,
         '-set', "BINDS='%s'" % bind_str,
-        command,
     ]
+    if verbose:
+        involucro_args.extend(["-set", "VERBOSE='1'"])
+    if conda_version is not None:
+        verbose = "--verbose" if verbose else "--quiet"
+        involucro_args.extend(["-set", "PREINSTALL='conda install %s --yes conda=%s'" % (verbose, conda_version)])
+    involucro_args.append(command)
     if test_files:
         test_bind = []
         for test_file in test_files:
@@ -165,7 +170,8 @@ def mull_targets(
 
 
 def context_from_args(args):
-    return InvolucroContext(involucro_bin=args.involucro_path)
+    verbose = "2" if not args.verbose else "3"
+    return InvolucroContext(involucro_bin=args.involucro_path, verbose=verbose)
 
 
 class InvolucroContext(installable.InstallableContext):
@@ -220,6 +226,8 @@ def add_build_arguments(parser):
                         help="Rebuild package even if already published.")
     parser.add_argument('--dry-run', dest='dry_run', action="store_true",
                         help='Just print commands instead of executing them.')
+    parser.add_argument('--verbose', dest='verbose', action="store_true",
+                        help='Cause process to be verbose.')
     parser.add_argument('-n', '--namespace', dest='namespace', default="mulled",
                         help='quay.io namespace.')
     parser.add_argument('-r', '--repository_template', dest='repository_template', default=DEFAULT_REPOSITORY_TEMPLATE,
@@ -228,6 +236,8 @@ def add_build_arguments(parser):
                         help='Target conda channel')
     parser.add_argument('--extra-channels', dest='extra_channels', default=",".join(DEFAULT_EXTRA_CHANNELS),
                         help='Dependent conda channels.')
+    parser.add_argument('--conda-version', dest="conda_version", default=None,
+                        help="Change to specified version of Conda before installing packages.")
 
 
 def add_single_image_arguments(parser):
@@ -277,6 +287,8 @@ def args_to_mull_targets_kwds(args):
         kwds["command"] = args.command
     if hasattr(args, "repository_template"):
         kwds["repository_template"] = args.repository_template
+    if hasattr(args, "conda_version"):
+        kwds["conda_version"] = args.conda_version
 
     kwds["involucro_context"] = context_from_args(args)
 

lib/galaxy/tools/deps/resolvers/conda.py

@@ -55,6 +55,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li
         'auto_install': False,
         'auto_init': True,
         'copy_dependencies': False,
+        'use_local': False,
     }
     _specification_pattern = re.compile(r"https\:\/\/anaconda.org\/\w+\/\w+")
 
@@ -84,6 +85,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li
             )
 
         copy_dependencies = _string_as_bool(get_option("copy_dependencies"))
+        use_local = _string_as_bool(get_option("use_local"))
         conda_exec = get_option("exec")
         debug = _string_as_bool(get_option("debug"))
         ensure_channels = get_option("ensure_channels")
@@ -102,7 +104,8 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li
             ensure_channels=ensure_channels,
             condarc_override=condarc_override,
             use_path_exec=use_path_exec,
-            copy_dependencies=copy_dependencies
+            copy_dependencies=copy_dependencies,
+            use_local=use_local,
         )
         self.ensure_channels = ensure_channels
 
@@ -111,6 +114,8 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li
         self.auto_init = _string_as_bool(get_option("auto_init"))
         self.conda_context = conda_context
         self.disabled = not galaxy.tools.deps.installable.ensure_installed(conda_context, install_conda, self.auto_init)
+        if self.auto_init and not self.disabled:
+            self.conda_context.ensure_conda_build_installed_if_needed()
         self.auto_install = auto_install
         self.copy_dependencies = copy_dependencies
 

lib/galaxy/tools/fetcher.py

@@ -0,0 +1,25 @@
+from galaxy.util import plugin_config
+
+
+class ToolLocationFetcher(object):
+
+    def __init__(self):
+        self.resolver_classes = self.__resolvers_dict()
+
+    def __resolvers_dict( self ):
+        import galaxy.tools.locations
+        return plugin_config.plugins_dict(galaxy.tools.locations, 'scheme')
+
+    def to_tool_path(self, path_or_uri_like, **kwds):
+        if "://" not in path_or_uri_like:
+            path = path_or_uri_like
+        else:
+            uri_like = path_or_uri_like
+            if ":" not in path_or_uri_like:
+                raise Exception("Invalid URI passed to get_tool_source")
+            scheme, rest = uri_like.split(":", 2)
+            if scheme not in self.resolver_classes:
+                raise Exception("Unknown tool scheme [%s] for URI [%s]" % (scheme, uri_like))
+            path = self.resolver_classes[scheme]().get_tool_source_path(uri_like)
+
+        return path

lib/galaxy/tools/loader_directory.py

@@ -119,7 +119,7 @@ def is_tool_load_error(obj):
     return obj is TOOL_LOAD_ERROR
 
 
-def looks_like_a_tool(path, invalid_names=[], enable_beta_formats=False):
+def looks_like_a_tool(path_or_uri_like, invalid_names=[], enable_beta_formats=False):
     """Quick check to see if a file looks like it may be a tool file.
 
     Whether true in a strict sense or not, lets say the intention and
@@ -130,6 +130,11 @@ def looks_like_a_tool(path, invalid_names=[], enable_beta_formats=False):
     invalid_names may be supplied in the context of the tool shed to quickly
     rule common tool shed XML files.
     """
+    path = resolved_path(path_or_uri_like)
+    if path is UNRESOLVED_URI:
+        # Assume the path maps to a real tool.
+        return True
+
     looks = False
 
     if os.path.basename(path) in invalid_names:
@@ -222,7 +227,12 @@ def looks_like_a_tool_cwl(path):
     return looks_like_a_cwl_artifact(path, classes=["CommandLineTool", "ExpressionTool"])
 
 
-def _find_tool_files(path, recursive, enable_beta_formats):
+def _find_tool_files(path_or_uri_like, recursive, enable_beta_formats):
+    path = resolved_path(path_or_uri_like)
+    if path is UNRESOLVED_URI:
+        # Pass the URI through and assume it maps to a real tool.
+        return [path_or_uri_like]
+
     is_file = not os.path.isdir(path)
     if not os.path.exists(path):
         raise Exception(PATH_DOES_NOT_EXIST_ERROR)
@@ -261,6 +271,19 @@ def _find_files(directory, pattern='*'):
     return matches
 
 
+UNRESOLVED_URI = object()
+
+
+def resolved_path(path_or_uri_like):
+    """If this is a simple file path, return the path else UNRESOLVED_URI."""
+    if "://" not in path_or_uri_like:
+        return path_or_uri_like
+    elif path_or_uri_like.startswith("file://"):
+        return path_or_uri_like[len("file://"):]
+    else:
+        return UNRESOLVED_URI
+
+
 BETA_TOOL_CHECKERS = {
     'yaml': looks_like_a_tool_yaml,
     'cwl': looks_like_a_tool_cwl,

lib/galaxy/tools/locations/__init__.py

@@ -0,0 +1,29 @@
+import os
+import tempfile
+
+from abc import (
+    ABCMeta,
+    abstractmethod,
+    abstractproperty,
+)
+
+import six
+
+
+@six.add_metaclass(ABCMeta)
+class ToolLocationResolver(object):
+    """Parse a URI-like string and return a ToolSource object."""
+
+    @abstractproperty
+    def scheme(self):
+        """Short label for the type of location resolver and URI scheme."""
+
+    @abstractmethod
+    def get_tool_source_path(self, uri_like):
+        """Return a local path for the uri_like string."""
+
+    def _temp_path(self, uri_like):
+        """Create an abstraction for this so we can configure and cache later."""
+        handle, filename = tempfile.mkstemp(suffix=uri_like.split("/")[-1])
+        os.close(handle)
+        return filename

lib/galaxy/tools/locations/dockstore.py

@@ -0,0 +1,69 @@
+try:
+    import requests
+except ImportError:
+    requests = None
+import yaml
+
+from six.moves.urllib.parse import quote
+
+from ..locations import (
+    ToolLocationResolver,
+)
+
+
+class DockStoreResolver(ToolLocationResolver):
+
+    scheme = "dockstore"
+
+    def get_tool_source_path(self, uri_like):
+        assert uri_like.startswith("dockstore://")
+        tool_id = uri_like[len("dockstore://"):]
+        if ":" in tool_id:
+            tool_id, version = tool_id.split(":", 1)
+        else:
+            tool_id, version = tool_id, "latest"
+        tmp_path = self._temp_path(uri_like + ".cwl")
+        cwl_str = _Ga4ghToolClient().get_tool_cwl(tool_id, version=version, as_string=True)
+        with open(tmp_path, "wb") as f:
+            f.write(cwl_str)
+        return tmp_path
+
+
+class _Ga4ghToolClient(object):
+
+    def __init__(self, base_url="https://www.dockstore.org:8443/api"):
+        self.base_url = base_url
+
+    def get_tools(self):
+        return self._requests.get("%s/ga4gh/v1/tools" % self.base_url)
+
+    def get_tool(self, tool_id):
+        url = "%s/ga4gh/v1/tools/%s" % (self.base_url, quote(tool_id, safe=''))
+        return self._requests.get(url)
+
+    def get_tool_version(self, tool_id, version="latest"):
+        url = "%s/ga4gh/v1/tools/%s/versions/%s" % (self.base_url, quote(tool_id, safe=''), version)
+        return self._requests.get(url)
+
+    def get_tool_descriptor(self, tool_id, version="latest", tool_type="CWL"):
+        url = "%s/ga4gh/v1/tools/%s/versions/%s/%s/descriptor" % (self.base_url, quote(tool_id, safe=''), version, tool_type)
+        return self._requests.get(url)
+
+    def get_tool_cwl(self, tool_id, version="latest", as_string=False):
+        tool_type = "CWL"
+        url = "%s/ga4gh/v1/tools/%s/versions/%s/%s/descriptor" % (self.base_url, quote(tool_id, safe=''), version, tool_type)
+        descriptor_response = self._requests.get(url)
+        descriptor_str = descriptor_response.json()["descriptor"]
+        if as_string:
+            return descriptor_str
+        else:
+            return yaml.load(descriptor_str)
+
+    @property
+    def _requests(self):
+        if requests is None:
+            raise Exception("requests Python library needs to be installed use GA4GH APIs")
+        return requests
+
+
+__all__ = ("DockStoreResolver",)

lib/galaxy/tools/locations/file.py

@@ -0,0 +1,12 @@
+from ..locations import (
+    ToolLocationResolver,
+)
+
+
+class HttpToolResolver(ToolLocationResolver):
+
+    scheme = "file"
+
+    def get_tool_source_path(self, uri_like):
+        assert uri_like.startswith("file://")
+        return uri_like[len("file://"):]

lib/galaxy/tools/locations/http.py

@@ -0,0 +1,26 @@
+from galaxy.util import download_to_file
+
+from ..locations import (
+    ToolLocationResolver,
+)
+
+
+class HttpToolResolver(ToolLocationResolver):
+
+    scheme = "http"
+
+    def __init__(self, **kwds):
+        pass
+
+    def get_tool_source_path(self, uri_like):
+        tmp_path = self._temp_path(uri_like)
+        download_to_file(uri_like, tmp_path)
+        return tmp_path
+
+
+class HttpsToolResolver(HttpToolResolver):
+
+    scheme = "https"
+
+
+__all__ = ("HttpToolResolver", "HttpsToolResolver")

lib/galaxy/tools/parser/cwl.py

@@ -18,16 +18,17 @@ log = logging.getLogger(__name__)
 
 class CwlToolSource(ToolSource):
 
-    def __init__(self, tool_file):
+    def __init__(self, tool_file, strict_cwl_validation=True):
         self._cwl_tool_file = tool_file
         self._id, _ = os.path.splitext(os.path.basename(tool_file))
         self._tool_proxy = None
         self._source_path = tool_file
+        self._strict_cwl_validation = strict_cwl_validation
 
     @property
     def tool_proxy(self):
         if self._tool_proxy is None:
-            self._tool_proxy = tool_proxy(self._source_path)
+            self._tool_proxy = tool_proxy(self._source_path, strict_cwl_validation=self._strict_cwl_validation)
         return self._tool_proxy
 
     def parse_tool_type(self):
@@ -95,7 +96,7 @@ class CwlToolSource(ToolSource):
         return "0.0.1"
 
     def parse_description(self):
-        return ""
+        return self.tool_proxy.description()
 
     def parse_input_pages(self):
         page_source = CwlPageSource(self.tool_proxy)

lib/galaxy/tools/parser/factory.py

@@ -13,10 +13,12 @@ from .interface import InputSource
 from .xml import XmlInputSource, XmlToolSource
 from .yaml import YamlToolSource
 
+from ..fetcher import ToolLocationFetcher
+
 log = logging.getLogger(__name__)
 
 
-def get_tool_source(config_file=None, xml_tree=None, enable_beta_formats=True):
+def get_tool_source(config_file=None, xml_tree=None, enable_beta_formats=True, tool_location_fetcher=None):
     """Return a ToolSource object corresponding to supplied source.
 
     The supplied source may be specified as a file path (using the config_file
@@ -27,6 +29,10 @@ def get_tool_source(config_file=None, xml_tree=None, enable_beta_formats=True):
     elif config_file is None:
         raise ValueError("get_tool_source called with invalid config_file None.")
 
+    if tool_location_fetcher is None:
+        tool_location_fetcher = ToolLocationFetcher()
+
+    config_file = tool_location_fetcher.to_tool_path(config_file)
     if not enable_beta_formats:
         tree = load_tool_xml(config_file)
         return XmlToolSource(tree, source_path=config_file)