diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index a44c2336527..0a35fef69d6 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -42,6 +42,7 @@ from galaxy.tools.deps import ( CachedDependencyManager, views ) +from galaxy.tools.fetcher import ToolLocationFetcher from galaxy.tools.parameters import ( check_param, params_from_strings, @@ -181,6 +182,7 @@ class ToolBox( BaseGalaxyToolBox ): def __init__( self, config_filenames, tool_root_dir, app, tool_conf_watcher=None ): self._reload_count = 0 + self.tool_location_fetcher = ToolLocationFetcher() super( ToolBox, self ).__init__( config_filenames=config_filenames, tool_root_dir=tool_root_dir, @@ -216,7 +218,11 @@ class ToolBox( BaseGalaxyToolBox ): def create_tool( self, config_file, repository_id=None, guid=None, **kwds ): try: - tool_source = get_tool_source( config_file, enable_beta_formats=getattr( self.app.config, "enable_beta_tool_formats", False ) ) + tool_source = get_tool_source( + config_file, + enable_beta_formats=getattr( self.app.config, "enable_beta_tool_formats", False ), + tool_location_fetcher=self.tool_location_fetcher, + ) except Exception as e: # capture and log parsing errors global_tool_errors.add_error(config_file, "Tool XML parsing", e) diff --git a/lib/galaxy/tools/cwl/cwltool_deps.py b/lib/galaxy/tools/cwl/cwltool_deps.py index a2a946c91a1..9c2518f03d9 100644 --- a/lib/galaxy/tools/cwl/cwltool_deps.py +++ b/lib/galaxy/tools/cwl/cwltool_deps.py @@ -17,6 +17,7 @@ try: workflow, job, process, + pathmapper, ) except (ImportError, SyntaxError): # Drop SyntaxError once cwltool supports Python 3 @@ -24,6 +25,7 @@ except (ImportError, SyntaxError): workflow = None job = None process = None + pathmapper = None try: from cwltool import load_tool @@ -41,6 +43,12 @@ except (ImportError, SyntaxError): # Drop SyntaxError once schema_salad supports Python 3 schema_salad = None +try: + from schema_salad import ref_resolver +except (ImportError, SyntaxError): + ref_resolver = None + + needs_shell_quoting = re.compile(r"""(^$|[\s|&;()<>\'"$@])""").search @@ -66,9 +74,11 @@ def ensure_cwltool_available(): __all__ = ( 'main', + 'ref_resolver', 'load_tool', 'workflow', 'process', + 'pathmapper', 'ensure_cwltool_available', 'schema_salad', 'shellescape', diff --git a/lib/galaxy/tools/cwl/parser.py b/lib/galaxy/tools/cwl/parser.py index 3e106440933..195fdf28e5e 100644 --- a/lib/galaxy/tools/cwl/parser.py +++ b/lib/galaxy/tools/cwl/parser.py @@ -18,11 +18,10 @@ from galaxy.util.odict import odict from .cwltool_deps import ( ensure_cwltool_available, - main, - workflow, + process, ) -from .schema import schema_loader +from .schema import non_strict_schema_loader, schema_loader log = logging.getLogger(__name__) @@ -42,36 +41,36 @@ SUPPORTED_WORKFLOW_REQUIREMENTS = SUPPORTED_TOOL_REQUIREMENTS + [ ] -def tool_proxy(tool_path): +def tool_proxy(tool_path, strict_cwl_validation=True): """ Provide a proxy object to cwltool data structures to just grab relevant data. """ ensure_cwltool_available() - tool = to_cwl_tool_object(tool_path) + tool = to_cwl_tool_object(tool_path, strict_cwl_validation=strict_cwl_validation) return tool -def workflow_proxy(workflow_path): +def workflow_proxy(workflow_path, strict_cwl_validation=True): ensure_cwltool_available() - workflow = to_cwl_workflow_object(workflow_path) + workflow = to_cwl_workflow_object(workflow_path, strict_cwl_validation=strict_cwl_validation) return workflow -def load_job_proxy(job_directory): +def load_job_proxy(job_directory, strict_cwl_validation=True): ensure_cwltool_available() job_objects_path = os.path.join(job_directory, JOB_JSON_FILE) job_objects = json.load(open(job_objects_path, "r")) tool_path = job_objects["tool_path"] job_inputs = job_objects["job_inputs"] output_dict = job_objects["output_dict"] - cwl_tool = tool_proxy(tool_path) + cwl_tool = tool_proxy(tool_path, strict_cwl_validation=strict_cwl_validation) cwl_job = cwl_tool.job_proxy(job_inputs, output_dict, job_directory=job_directory) return cwl_job -def to_cwl_tool_object(tool_path): +def to_cwl_tool_object(tool_path, strict_cwl_validation=True): proxy_class = None - cwl_tool = schema_loader.tool(path=tool_path) + cwl_tool = _schema_loader(strict_cwl_validation).tool(path=tool_path) if isinstance(cwl_tool, int): raise Exception("Failed to load tool.") @@ -93,10 +92,9 @@ def to_cwl_tool_object(tool_path): return proxy -def to_cwl_workflow_object(workflow_path): +def to_cwl_workflow_object(workflow_path, strict_cwl_validation=None): proxy_class = WorkflowProxy - make_tool = workflow.defaultMakeTool - cwl_workflow = main.load_tool(workflow_path, False, False, make_tool, False) + cwl_workflow = _schema_loader(strict_cwl_validation).tool(path=workflow_path) raw_workflow = cwl_workflow.tool check_requirements(raw_workflow, tool=False) @@ -104,6 +102,11 @@ def to_cwl_workflow_object(workflow_path): return proxy +def _schema_loader(strict_cwl_validation): + target_schema_loader = schema_loader if strict_cwl_validation else non_strict_schema_loader + return target_schema_loader + + def check_requirements(rec, tool=True): if isinstance(rec, dict): if "requirements" in rec: @@ -237,14 +240,21 @@ class JobProxy(object): def _ensure_cwl_job_initialized(self): if self._cwl_job is None: + self._cwl_job = next(self._tool_proxy._tool.job( self._input_dict, self._output_callback, basedir=self._job_directory, + select_resources=self._select_resources, use_container=False )) self._is_command_line_job = hasattr(self._cwl_job, "command_line") + def _select_resources(self, request): + new_request = request.copy() + new_request["cores"] = "$GALAXY_SLOTS" + return new_request + @property def command_line(self): if self.is_command_line_job: @@ -324,6 +334,12 @@ class JobProxy(object): safe_makedirs(secondary_files_dir) return secondary_files_dir + def stage_files(self): + cwl_job = self.cwl_job() + if hasattr(cwl_job, "pathmapper"): + process.stageFiles(self.cwl_job().pathmapper, os.symlink, ignoreWritable=True) + # else: expression tools do not have a path mapper. + @staticmethod def _job_file(job_directory): return os.path.join(job_directory, JOB_JSON_FILE) diff --git a/lib/galaxy/tools/cwl/representation.py b/lib/galaxy/tools/cwl/representation.py index 140708b0f64..fde19ffbf1f 100644 --- a/lib/galaxy/tools/cwl/representation.py +++ b/lib/galaxy/tools/cwl/representation.py @@ -19,6 +19,7 @@ GALAXY_TO_CWL_TYPES = { 'float': 'float', 'data': 'File', 'boolean': 'boolean', + 'text': 'text' } @@ -56,7 +57,8 @@ def to_cwl_job(tool, param_dict, local_working_directory): os.symlink(secondary_file_path, new_input_path + secondary_file_name) path = new_input_path - return {"path": path, "class": "File"} + return {"location": path, + "class": "File"} elif cwl_type == "integer": return int(str(param_dict_value)) elif cwl_type == "long": @@ -67,7 +69,7 @@ def to_cwl_job(tool, param_dict, local_working_directory): return float(str(param_dict_value)) elif cwl_type == "boolean": return string_as_bool(param_dict_value) - elif cwl_type == "string": + elif cwl_type == "text": return str(param_dict_value) elif cwl_type == "json": raw_value = param_dict_value.value @@ -94,9 +96,6 @@ def to_cwl_job(tool, param_dict, local_working_directory): else: input_json[input_name] = simple_value(input, param_dict[input_name]) - input_json["allocatedResources"] = { - "cpu": "$GALAXY_SLOTS", - } return input_json diff --git a/lib/galaxy/tools/cwl/runtime_actions.py b/lib/galaxy/tools/cwl/runtime_actions.py index 07a4b2c5731..c3814b7d29d 100644 --- a/lib/galaxy/tools/cwl/runtime_actions.py +++ b/lib/galaxy/tools/cwl/runtime_actions.py @@ -2,6 +2,7 @@ import json import os import shutil +from .cwltool_deps import ref_resolver from .parser import ( JOB_JSON_FILE, load_job_proxy, @@ -17,13 +18,16 @@ def handle_outputs(job_directory=None): if not os.path.exists(cwl_job_file): # Not a CWL job, just continue return - job_proxy = load_job_proxy(job_directory) + # So we only need to do strict validation when the tool was loaded, + # no reason to do it again during job execution - so this shortcut + # allows us to not need Galaxy's full configuration on job nodes. + job_proxy = load_job_proxy(job_directory, strict_cwl_validation=False) tool_working_directory = os.path.join(job_directory, "working") outputs = job_proxy.collect_outputs(tool_working_directory) for output_name, output in outputs.items(): - target_path = job_proxy.output_path( output_name ) - if isinstance(output, dict) and "path" in output: - output_path = output["path"] + target_path = job_proxy.output_path(output_name) + if isinstance(output, dict) and "location" in output: + output_path = ref_resolver.uri_file_path(output["location"]) if output["class"] != "File": open("galaxy.json", "w").write(json.dump({ "dataset_id": job_proxy.output_id(output_name), @@ -33,7 +37,7 @@ def handle_outputs(job_directory=None): shutil.move(output_path, target_path) for secondary_file in output.get("secondaryFiles", []): # TODO: handle nested files... - secondary_file_path = secondary_file["path"] + secondary_file_path = ref_resolver.uri_file_path(secondary_file["location"]) assert secondary_file_path.startswith(output_path) secondary_file_name = secondary_file_path[len(output_path):] secondary_files_dir = job_proxy.output_secondary_files_dir( diff --git a/lib/galaxy/tools/cwl/schema.py b/lib/galaxy/tools/cwl/schema.py index fb24b784724..6b3e1e49ccd 100644 --- a/lib/galaxy/tools/cwl/schema.py +++ b/lib/galaxy/tools/cwl/schema.py @@ -24,11 +24,8 @@ class SchemaLoader(object): @property def raw_document_loader(self): - if self._raw_document_loader is None: - ensure_cwltool_available() - self._raw_document_loader = schema_salad.ref_resolver.Loader({"cwl": "https://w3id.org/cwl/cwl#", "id": "@id"}) - - return self._raw_document_loader + ensure_cwltool_available() + return schema_salad.ref_resolver.Loader({"cwl": "https://w3id.org/cwl/cwl#", "id": "@id"}) def raw_process_reference(self, path): uri = "file://" + os.path.abspath(path) @@ -71,3 +68,4 @@ class SchemaLoader(object): schema_loader = SchemaLoader() +non_strict_schema_loader = SchemaLoader(strict=False) diff --git a/lib/galaxy/tools/deps/conda_util.py b/lib/galaxy/tools/deps/conda_util.py index eccb00d37f8..00ca742f24e 100644 --- a/lib/galaxy/tools/deps/conda_util.py +++ b/lib/galaxy/tools/deps/conda_util.py @@ -30,6 +30,10 @@ VERSIONED_ENV_DIR_NAME = re.compile(r"__(.*)@(.*)") UNVERSIONED_ENV_DIR_NAME = re.compile(r"__(.*)@_uv_") USE_PATH_EXEC_DEFAULT = False CONDA_VERSION = "4.2.13" +# 2.1.5 is incompatible with 4.2 Conda because of +# https://github.com/conda/conda-build/pull/1766 +CONDA_BUILD_VERSION = "2.1.4" +USE_LOCAL_DEFAULT = False def conda_link(): @@ -45,7 +49,16 @@ def find_conda_prefix(conda_prefix=None): for Miniconda installs. """ if conda_prefix is None: - return os.path.join(os.path.expanduser("~"), "miniconda2") + home = os.path.expanduser("~") + miniconda_2_dest = os.path.join(home, "miniconda2") + miniconda_3_dest = os.path.join(home, "miniconda3") + # Prefer miniconda3 install if both available + if os.path.exists(miniconda_3_dest): + return miniconda_3_dest + elif os.path.exists(miniconda_2_dest): + return miniconda_2_dest + else: + return miniconda_3_dest return conda_prefix @@ -54,7 +67,8 @@ class CondaContext(installable.InstallableContext): def __init__(self, conda_prefix=None, conda_exec=None, shell_exec=None, debug=False, ensure_channels='', - condarc_override=None, use_path_exec=USE_PATH_EXEC_DEFAULT, copy_dependencies=False): + condarc_override=None, use_path_exec=USE_PATH_EXEC_DEFAULT, + copy_dependencies=False, use_local=USE_LOCAL_DEFAULT): self.condarc_override = condarc_override if not conda_exec and use_path_exec: conda_exec = commands.which("conda") @@ -84,18 +98,27 @@ class CondaContext(installable.InstallableContext): self.ensured_channels = False self._conda_version = None self._miniconda_version = None + self._conda_build_available = None + self.use_local = use_local @property def conda_version(self): if self._conda_version is None: - self._guess_conda_version() + self._guess_conda_properties() return self._conda_version - def _guess_conda_version(self): + @property + def conda_build_available(self): + if self._conda_build_available is None: + self._guess_conda_properties() + return self._conda_build_available + + def _guess_conda_properties(self): conda_meta_path = self._conda_meta_path # Perhaps we should call "conda info --json" and parse it but for now we are going # to assume the default. conda_version = LooseVersion(CONDA_VERSION) + conda_build_available = False miniconda_version = "3" if os.path.exists(conda_meta_path): @@ -110,9 +133,12 @@ class CondaContext(installable.InstallableContext): conda_version = LooseVersion(version) if package == "python" and version.startswith("2"): miniconda_version = "2" + if package == "conda-build": + conda_build_available = True self._conda_version = conda_version self._miniconda_version = miniconda_version + self._conda_build_available = conda_build_available @property def _conda_meta_path(self): @@ -135,6 +161,14 @@ class CondaContext(installable.InstallableContext): if changed: self.save_condarc(conda_conf) + def ensure_conda_build_installed_if_needed(self): + if self.use_local and not self.conda_build_available: + conda_targets = [CondaTarget("conda-build", version=CONDA_BUILD_VERSION)] + # Cannot use --use-local during installation fo conda-build. + return install_conda_targets(conda_targets, env_name=None, conda_context=self, allow_local=False) + else: + return 0 + def conda_info(self): if self.conda_exec is not None: info_out = commands.execute([self.conda_exec, "info", "--json"]) @@ -229,10 +263,12 @@ class CondaContext(installable.InstallableContext): finally: shutil.rmtree(env['HOME'], ignore_errors=True) - def exec_create(self, args): + def exec_create(self, args, allow_local=True): create_base_args = [ "-y" ] + if allow_local and self.use_local: + create_base_args.extend(["--use-local"]) create_base_args.extend(args) return self.exec_command("create", create_base_args) @@ -246,10 +282,12 @@ class CondaContext(installable.InstallableContext): remove_base_args.extend(args) return self.exec_command("env", remove_base_args) - def exec_install(self, args): + def exec_install(self, args, allow_local=True): install_base_args = [ "-y" ] + if allow_local and self.use_local: + install_base_args.extend(["--use-local"]) install_base_args.extend(args) return self.exec_command("install", install_base_args) @@ -390,13 +428,18 @@ def hash_conda_packages(conda_packages, conda_target=None): # shell makes sense for planemo, in Galaxy this should just execute # these commands as Python -def install_conda(conda_context=None): +def install_conda(conda_context=None, force_conda_build=False): conda_context = _ensure_conda_context(conda_context) f, script_path = tempfile.mkstemp(suffix=".sh", prefix="conda_install") os.close(f) download_cmd = " ".join(commands.download_command(conda_link(), to=script_path, quote_url=True)) install_cmd = "bash '%s' -b -p '%s'" % (script_path, conda_context.conda_prefix) - fix_version_cmd = "%s install -y -q conda=%s " % (os.path.join(conda_context.conda_prefix, 'bin/conda'), CONDA_VERSION) + package_targets = [ + "conda=%s" % CONDA_VERSION, + ] + if force_conda_build or conda_context.use_local: + package_targets.append("conda-build=%s" % CONDA_BUILD_VERSION) + fix_version_cmd = "%s install -y -q %s " % (os.path.join(conda_context.conda_prefix, 'bin/conda'), " ".join(package_targets)) full_command = "%s && %s && %s" % (download_cmd, install_cmd, fix_version_cmd) try: log.info("Installing Conda, this may take several minutes.") @@ -406,7 +449,7 @@ def install_conda(conda_context=None): os.remove(script_path) -def install_conda_targets(conda_targets, env_name=None, conda_context=None): +def install_conda_targets(conda_targets, env_name=None, conda_context=None, allow_local=True): conda_context = _ensure_conda_context(conda_context) conda_context.ensure_channels_configured() if env_name is not None: @@ -415,9 +458,9 @@ def install_conda_targets(conda_targets, env_name=None, conda_context=None): ] for conda_target in conda_targets: create_args.append(conda_target.package_specifier) - return conda_context.exec_create(create_args) + return conda_context.exec_create(create_args, allow_local=allow_local) else: - return conda_context.exec_install([t.package_specifier for t in conda_targets]) + return conda_context.exec_install([t.package_specifier for t in conda_targets], allow_local=allow_local) def install_conda_target(conda_target, conda_context=None, skip_environment=False): diff --git a/lib/galaxy/tools/deps/mulled/invfile.lua b/lib/galaxy/tools/deps/mulled/invfile.lua index cc5260ba7a7..693d7ecf6f9 100644 --- a/lib/galaxy/tools/deps/mulled/invfile.lua +++ b/lib/galaxy/tools/deps/mulled/invfile.lua @@ -40,19 +40,49 @@ for i = 1, #test_binds_table do table.insert(test_bind_args, test_binds_table[i]) end +local conda_image = VAR.CONDA_IMAGE +if conda_image == '' then + conda_image = 'continuumio/miniconda:latest' +end + +local destination_base_image = VAR.DEST_BASE_IMAGE +if destination_base_image == '' then + destination_base_image = 'bgruening/busybox-bash:0.1' +end + +local verbose = VAR.VERBOSE +if verbose == '' then + verbose = '--quiet' +else + verbose = '--verbose' +end + +local preinstall = VAR.PREINSTALL +if preinstall ~= '' then + preinstall = preinstall .. ' && ' +end + +local postinstall = VAR.POSTINSTALL +if postinstall ~= '' then + postinstall = '&&' .. postinstall +end + inv.task('build') - .using('continuumio/miniconda:latest') + .using(conda_image) .withHostConfig({binds = {"build:/data"}}) .run('rm', '-rf', '/data/dist') - .using('continuumio/miniconda:latest') + .using(conda_image) .withHostConfig({binds = bind_args}) - .run('/bin/sh', '-c', 'conda install ' + .run('/bin/sh', '-c', preinstall + .. 'conda install ' .. channel_args .. ' ' .. target_args - .. ' -p /usr/local --copy --yes --quiet') + .. ' -p /usr/local --copy --yes ' + .. verbose + .. postinstall) .wrap('build/dist') .at('/usr/local') - .inImage('bgruening/busybox-bash:0.1') + .inImage(destination_base_image) .as(repo) if VAR.TEST_BINDS == '' then diff --git a/lib/galaxy/tools/deps/mulled/mulled_build.py b/lib/galaxy/tools/deps/mulled/mulled_build.py index fb4e7b293f6..f2cc20f304f 100644 --- a/lib/galaxy/tools/deps/mulled/mulled_build.py +++ b/lib/galaxy/tools/deps/mulled/mulled_build.py @@ -116,7 +116,7 @@ def mull_targets( command="build", channels=DEFAULT_CHANNELS, namespace="mulled", test='true', test_files=None, image_build=None, name_override=None, repository_template=DEFAULT_REPOSITORY_TEMPLATE, dry_run=False, - binds=DEFAULT_BINDS + conda_version=None, verbose=False, binds=DEFAULT_BINDS ): targets = list(targets) if involucro_context is None: @@ -143,8 +143,13 @@ def mull_targets( '-set', "TARGETS='%s'" % target_str, '-set', "REPO='%s'" % repo, '-set', "BINDS='%s'" % bind_str, - command, ] + if verbose: + involucro_args.extend(["-set", "VERBOSE='1'"]) + if conda_version is not None: + verbose = "--verbose" if verbose else "--quiet" + involucro_args.extend(["-set", "PREINSTALL='conda install %s --yes conda=%s'" % (verbose, conda_version)]) + involucro_args.append(command) if test_files: test_bind = [] for test_file in test_files: @@ -165,7 +170,8 @@ def mull_targets( def context_from_args(args): - return InvolucroContext(involucro_bin=args.involucro_path) + verbose = "2" if not args.verbose else "3" + return InvolucroContext(involucro_bin=args.involucro_path, verbose=verbose) class InvolucroContext(installable.InstallableContext): @@ -220,6 +226,8 @@ def add_build_arguments(parser): help="Rebuild package even if already published.") parser.add_argument('--dry-run', dest='dry_run', action="store_true", help='Just print commands instead of executing them.') + parser.add_argument('--verbose', dest='verbose', action="store_true", + help='Cause process to be verbose.') parser.add_argument('-n', '--namespace', dest='namespace', default="mulled", help='quay.io namespace.') parser.add_argument('-r', '--repository_template', dest='repository_template', default=DEFAULT_REPOSITORY_TEMPLATE, @@ -228,6 +236,8 @@ def add_build_arguments(parser): help='Target conda channel') parser.add_argument('--extra-channels', dest='extra_channels', default=",".join(DEFAULT_EXTRA_CHANNELS), help='Dependent conda channels.') + parser.add_argument('--conda-version', dest="conda_version", default=None, + help="Change to specified version of Conda before installing packages.") def add_single_image_arguments(parser): @@ -277,6 +287,8 @@ def args_to_mull_targets_kwds(args): kwds["command"] = args.command if hasattr(args, "repository_template"): kwds["repository_template"] = args.repository_template + if hasattr(args, "conda_version"): + kwds["conda_version"] = args.conda_version kwds["involucro_context"] = context_from_args(args) diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index 949a237694e..a74b219541e 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -55,6 +55,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li 'auto_install': False, 'auto_init': True, 'copy_dependencies': False, + 'use_local': False, } _specification_pattern = re.compile(r"https\:\/\/anaconda.org\/\w+\/\w+") @@ -84,6 +85,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li ) copy_dependencies = _string_as_bool(get_option("copy_dependencies")) + use_local = _string_as_bool(get_option("use_local")) conda_exec = get_option("exec") debug = _string_as_bool(get_option("debug")) ensure_channels = get_option("ensure_channels") @@ -102,7 +104,8 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li ensure_channels=ensure_channels, condarc_override=condarc_override, use_path_exec=use_path_exec, - copy_dependencies=copy_dependencies + copy_dependencies=copy_dependencies, + use_local=use_local, ) self.ensure_channels = ensure_channels @@ -111,6 +114,8 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li self.auto_init = _string_as_bool(get_option("auto_init")) self.conda_context = conda_context self.disabled = not galaxy.tools.deps.installable.ensure_installed(conda_context, install_conda, self.auto_init) + if self.auto_init and not self.disabled: + self.conda_context.ensure_conda_build_installed_if_needed() self.auto_install = auto_install self.copy_dependencies = copy_dependencies diff --git a/lib/galaxy/tools/fetcher.py b/lib/galaxy/tools/fetcher.py new file mode 100644 index 00000000000..3466c1a5212 --- /dev/null +++ b/lib/galaxy/tools/fetcher.py @@ -0,0 +1,25 @@ +from galaxy.util import plugin_config + + +class ToolLocationFetcher(object): + + def __init__(self): + self.resolver_classes = self.__resolvers_dict() + + def __resolvers_dict( self ): + import galaxy.tools.locations + return plugin_config.plugins_dict(galaxy.tools.locations, 'scheme') + + def to_tool_path(self, path_or_uri_like, **kwds): + if "://" not in path_or_uri_like: + path = path_or_uri_like + else: + uri_like = path_or_uri_like + if ":" not in path_or_uri_like: + raise Exception("Invalid URI passed to get_tool_source") + scheme, rest = uri_like.split(":", 2) + if scheme not in self.resolver_classes: + raise Exception("Unknown tool scheme [%s] for URI [%s]" % (scheme, uri_like)) + path = self.resolver_classes[scheme]().get_tool_source_path(uri_like) + + return path diff --git a/lib/galaxy/tools/loader_directory.py b/lib/galaxy/tools/loader_directory.py index bf78802d480..642f3d5ca9a 100644 --- a/lib/galaxy/tools/loader_directory.py +++ b/lib/galaxy/tools/loader_directory.py @@ -119,7 +119,7 @@ def is_tool_load_error(obj): return obj is TOOL_LOAD_ERROR -def looks_like_a_tool(path, invalid_names=[], enable_beta_formats=False): +def looks_like_a_tool(path_or_uri_like, invalid_names=[], enable_beta_formats=False): """Quick check to see if a file looks like it may be a tool file. Whether true in a strict sense or not, lets say the intention and @@ -130,6 +130,11 @@ def looks_like_a_tool(path, invalid_names=[], enable_beta_formats=False): invalid_names may be supplied in the context of the tool shed to quickly rule common tool shed XML files. """ + path = resolved_path(path_or_uri_like) + if path is UNRESOLVED_URI: + # Assume the path maps to a real tool. + return True + looks = False if os.path.basename(path) in invalid_names: @@ -222,7 +227,12 @@ def looks_like_a_tool_cwl(path): return looks_like_a_cwl_artifact(path, classes=["CommandLineTool", "ExpressionTool"]) -def _find_tool_files(path, recursive, enable_beta_formats): +def _find_tool_files(path_or_uri_like, recursive, enable_beta_formats): + path = resolved_path(path_or_uri_like) + if path is UNRESOLVED_URI: + # Pass the URI through and assume it maps to a real tool. + return [path_or_uri_like] + is_file = not os.path.isdir(path) if not os.path.exists(path): raise Exception(PATH_DOES_NOT_EXIST_ERROR) @@ -261,6 +271,19 @@ def _find_files(directory, pattern='*'): return matches +UNRESOLVED_URI = object() + + +def resolved_path(path_or_uri_like): + """If this is a simple file path, return the path else UNRESOLVED_URI.""" + if "://" not in path_or_uri_like: + return path_or_uri_like + elif path_or_uri_like.startswith("file://"): + return path_or_uri_like[len("file://"):] + else: + return UNRESOLVED_URI + + BETA_TOOL_CHECKERS = { 'yaml': looks_like_a_tool_yaml, 'cwl': looks_like_a_tool_cwl, diff --git a/lib/galaxy/tools/locations/__init__.py b/lib/galaxy/tools/locations/__init__.py new file mode 100644 index 00000000000..91b7a6f9f87 --- /dev/null +++ b/lib/galaxy/tools/locations/__init__.py @@ -0,0 +1,29 @@ +import os +import tempfile + +from abc import ( + ABCMeta, + abstractmethod, + abstractproperty, +) + +import six + + +@six.add_metaclass(ABCMeta) +class ToolLocationResolver(object): + """Parse a URI-like string and return a ToolSource object.""" + + @abstractproperty + def scheme(self): + """Short label for the type of location resolver and URI scheme.""" + + @abstractmethod + def get_tool_source_path(self, uri_like): + """Return a local path for the uri_like string.""" + + def _temp_path(self, uri_like): + """Create an abstraction for this so we can configure and cache later.""" + handle, filename = tempfile.mkstemp(suffix=uri_like.split("/")[-1]) + os.close(handle) + return filename diff --git a/lib/galaxy/tools/locations/dockstore.py b/lib/galaxy/tools/locations/dockstore.py new file mode 100644 index 00000000000..b379c23c38f --- /dev/null +++ b/lib/galaxy/tools/locations/dockstore.py @@ -0,0 +1,69 @@ +try: + import requests +except ImportError: + requests = None +import yaml + +from six.moves.urllib.parse import quote + +from ..locations import ( + ToolLocationResolver, +) + + +class DockStoreResolver(ToolLocationResolver): + + scheme = "dockstore" + + def get_tool_source_path(self, uri_like): + assert uri_like.startswith("dockstore://") + tool_id = uri_like[len("dockstore://"):] + if ":" in tool_id: + tool_id, version = tool_id.split(":", 1) + else: + tool_id, version = tool_id, "latest" + tmp_path = self._temp_path(uri_like + ".cwl") + cwl_str = _Ga4ghToolClient().get_tool_cwl(tool_id, version=version, as_string=True) + with open(tmp_path, "wb") as f: + f.write(cwl_str) + return tmp_path + + +class _Ga4ghToolClient(object): + + def __init__(self, base_url="https://www.dockstore.org:8443/api"): + self.base_url = base_url + + def get_tools(self): + return self._requests.get("%s/ga4gh/v1/tools" % self.base_url) + + def get_tool(self, tool_id): + url = "%s/ga4gh/v1/tools/%s" % (self.base_url, quote(tool_id, safe='')) + return self._requests.get(url) + + def get_tool_version(self, tool_id, version="latest"): + url = "%s/ga4gh/v1/tools/%s/versions/%s" % (self.base_url, quote(tool_id, safe=''), version) + return self._requests.get(url) + + def get_tool_descriptor(self, tool_id, version="latest", tool_type="CWL"): + url = "%s/ga4gh/v1/tools/%s/versions/%s/%s/descriptor" % (self.base_url, quote(tool_id, safe=''), version, tool_type) + return self._requests.get(url) + + def get_tool_cwl(self, tool_id, version="latest", as_string=False): + tool_type = "CWL" + url = "%s/ga4gh/v1/tools/%s/versions/%s/%s/descriptor" % (self.base_url, quote(tool_id, safe=''), version, tool_type) + descriptor_response = self._requests.get(url) + descriptor_str = descriptor_response.json()["descriptor"] + if as_string: + return descriptor_str + else: + return yaml.load(descriptor_str) + + @property + def _requests(self): + if requests is None: + raise Exception("requests Python library needs to be installed use GA4GH APIs") + return requests + + +__all__ = ("DockStoreResolver",) diff --git a/lib/galaxy/tools/locations/file.py b/lib/galaxy/tools/locations/file.py new file mode 100644 index 00000000000..3d25769783a --- /dev/null +++ b/lib/galaxy/tools/locations/file.py @@ -0,0 +1,12 @@ +from ..locations import ( + ToolLocationResolver, +) + + +class HttpToolResolver(ToolLocationResolver): + + scheme = "file" + + def get_tool_source_path(self, uri_like): + assert uri_like.startswith("file://") + return uri_like[len("file://"):] diff --git a/lib/galaxy/tools/locations/http.py b/lib/galaxy/tools/locations/http.py new file mode 100644 index 00000000000..d03ef3854da --- /dev/null +++ b/lib/galaxy/tools/locations/http.py @@ -0,0 +1,26 @@ +from galaxy.util import download_to_file + +from ..locations import ( + ToolLocationResolver, +) + + +class HttpToolResolver(ToolLocationResolver): + + scheme = "http" + + def __init__(self, **kwds): + pass + + def get_tool_source_path(self, uri_like): + tmp_path = self._temp_path(uri_like) + download_to_file(uri_like, tmp_path) + return tmp_path + + +class HttpsToolResolver(HttpToolResolver): + + scheme = "https" + + +__all__ = ("HttpToolResolver", "HttpsToolResolver") diff --git a/lib/galaxy/tools/parser/cwl.py b/lib/galaxy/tools/parser/cwl.py index 65c33d80146..a981bafdb07 100644 --- a/lib/galaxy/tools/parser/cwl.py +++ b/lib/galaxy/tools/parser/cwl.py @@ -18,16 +18,17 @@ log = logging.getLogger(__name__) class CwlToolSource(ToolSource): - def __init__(self, tool_file): + def __init__(self, tool_file, strict_cwl_validation=True): self._cwl_tool_file = tool_file self._id, _ = os.path.splitext(os.path.basename(tool_file)) self._tool_proxy = None self._source_path = tool_file + self._strict_cwl_validation = strict_cwl_validation @property def tool_proxy(self): if self._tool_proxy is None: - self._tool_proxy = tool_proxy(self._source_path) + self._tool_proxy = tool_proxy(self._source_path, strict_cwl_validation=self._strict_cwl_validation) return self._tool_proxy def parse_tool_type(self): @@ -95,7 +96,7 @@ class CwlToolSource(ToolSource): return "0.0.1" def parse_description(self): - return "" + return self.tool_proxy.description() def parse_input_pages(self): page_source = CwlPageSource(self.tool_proxy) diff --git a/lib/galaxy/tools/parser/factory.py b/lib/galaxy/tools/parser/factory.py index d38949f8498..69a5dbd7477 100644 --- a/lib/galaxy/tools/parser/factory.py +++ b/lib/galaxy/tools/parser/factory.py @@ -13,10 +13,12 @@ from .interface import InputSource from .xml import XmlInputSource, XmlToolSource from .yaml import YamlToolSource +from ..fetcher import ToolLocationFetcher + log = logging.getLogger(__name__) -def get_tool_source(config_file=None, xml_tree=None, enable_beta_formats=True): +def get_tool_source(config_file=None, xml_tree=None, enable_beta_formats=True, tool_location_fetcher=None): """Return a ToolSource object corresponding to supplied source. The supplied source may be specified as a file path (using the config_file @@ -27,6 +29,10 @@ def get_tool_source(config_file=None, xml_tree=None, enable_beta_formats=True): elif config_file is None: raise ValueError("get_tool_source called with invalid config_file None.") + if tool_location_fetcher is None: + tool_location_fetcher = ToolLocationFetcher() + + config_file = tool_location_fetcher.to_tool_path(config_file) if not enable_beta_formats: tree = load_tool_xml(config_file) return XmlToolSource(tree, source_path=config_file)