diff --git a/client/src/components/Workflow/Invocations.test.js b/client/src/components/Workflow/Invocations.test.js index 4ece7034cd7..8b1271e2bfe 100644 --- a/client/src/components/Workflow/Invocations.test.js +++ b/client/src/components/Workflow/Invocations.test.js @@ -96,6 +96,9 @@ describe("Invocations.vue", () => { propsData, computed: { getWorkflowNameByInstanceId: (state) => (id) => "workflow name", + getStoredWorkflowIdByInstanceId: (state) => (id) => { + return "workflowId"; + }, getWorkflowByInstanceId: (state) => (id) => { return { id: "workflowId" }; }, diff --git a/client/src/components/Workflow/Invocations.vue b/client/src/components/Workflow/Invocations.vue index 54dcca1bdaf..580677a7093 100644 --- a/client/src/components/Workflow/Invocations.vue +++ b/client/src/components/Workflow/Invocations.vue @@ -65,7 +65,10 @@ (workflowId) => { + const storedWorkflow = state.workflowsByInstanceId[workflowId]; + return storedWorkflow?.id; + }, }; const actions = { diff --git a/doc/source/admin/nginx.md b/doc/source/admin/nginx.md index f30b3517d1e..01cb555953f 100644 --- a/doc/source/admin/nginx.md +++ b/doc/source/admin/nginx.md @@ -442,7 +442,7 @@ galaxy: Galaxy creates zip archives when downloading multiple datasets from a history or a dataset library. While this works fine for small datasets and few users, nginx can handle the creation of zip archives more efficiently using [mod-zip](https://www.nginx.com/resources/wiki/modules/zip/). -To use this feature, install nginx with mod-zip enabled, provide the file locations from which +To use this feature, install nginx with mod-zip enabled (requires or a newer), provide the file locations from which nginx should serve files and edit `galaxy.yml` and make the following changes before restarting Galaxy: ```yaml diff --git a/lib/galaxy/config/sample/job_conf.xml.sample_advanced b/lib/galaxy/config/sample/job_conf.xml.sample_advanced index 5ac8d532c5e..9dab3586f57 100644 --- a/lib/galaxy/config/sample/job_conf.xml.sample_advanced +++ b/lib/galaxy/config/sample/job_conf.xml.sample_advanced @@ -606,6 +606,11 @@ + + diff --git a/lib/galaxy/datatypes/goldenpath.py b/lib/galaxy/datatypes/goldenpath.py index 11e46a37802..b21ddb3b07b 100755 --- a/lib/galaxy/datatypes/goldenpath.py +++ b/lib/galaxy/datatypes/goldenpath.py @@ -1,4 +1,5 @@ import abc +import logging import os from typing import ( Set, @@ -12,6 +13,8 @@ from galaxy.datatypes.sniff import ( ) from .tabular import Tabular +log = logging.getLogger(__name__) + @build_sniff_from_prefix class GoldenPath(Tabular): @@ -600,4 +603,6 @@ class AGPGapLine(AGPLine): ) else: if "na" in all_evidence: - raise AGPError(self.fname, self.line_number, "'na' is invalid linkage evidence when asserting linkage") + log.warning( + AGPError(self.fname, self.line_number, "'na' is invalid linkage evidence when asserting linkage") + ) diff --git a/lib/galaxy/datatypes/test/eg2.agp b/lib/galaxy/datatypes/test/eg2.agp index 57dea216b7a..d22c2379804 100644 --- a/lib/galaxy/datatypes/test/eg2.agp +++ b/lib/galaxy/datatypes/test/eg2.agp @@ -53,4 +53,4 @@ chrY 11824573 11874572 43 N 50000 contig no na chrY 11874573 11878623 44 W EG1_scaffold22 1 4051 + chrY 11878624 11928623 45 N 50000 contig no na chrY 11928624 11933053 46 W EG1_scaffold23 1 4430 + -chrY 11933054 11943053 47 N 10000 telomere no na +chrY 11933054 11943053 47 N 10000 telomere yes na diff --git a/lib/galaxy/managers/base.py b/lib/galaxy/managers/base.py index 4aa30a3f95f..3547ec19cca 100644 --- a/lib/galaxy/managers/base.py +++ b/lib/galaxy/managers/base.py @@ -28,6 +28,7 @@ attribute change to a model object. import datetime import logging import re +from functools import partial from typing import ( Any, Callable, @@ -68,6 +69,7 @@ log = logging.getLogger(__name__) class ParsedFilter(NamedTuple): filter_type: str # orm_function, function, or orm filter: Any + case_insensitive: bool = False parsed_filter = ParsedFilter @@ -1169,6 +1171,7 @@ class ModelFilterParser(HasAModelManager): allowed_ops = column_map["op"] if op not in allowed_ops: return None + converted_op = self._convert_op_string_to_fn(column, op) if not converted_op: return None @@ -1181,9 +1184,12 @@ class ModelFilterParser(HasAModelManager): val_parser = val_parser.get(op) if val_parser: val = val_parser(val) + if op == "contains": + # Do we want to make this configurable ? + val = val.lower() orm_filter = converted_op(val) - return self.parsed_filter(filter_type="orm", filter=orm_filter) + return self.parsed_filter(filter_type="orm", filter=orm_filter, case_insensitive=op == "contains") #: these are the easier/shorter string equivalents to the python operator fn names that need '__' around them UNDERSCORED_OPS = ("lt", "le", "eq", "ne", "ge", "gt") @@ -1205,6 +1211,8 @@ class ModelFilterParser(HasAModelManager): op_fn = getattr(column, fn_name, None) if not op_fn or not callable(op_fn): return None + if op_string == "contains": + op_fn = partial(op_fn, autoescape=True) return op_fn # ---- preset fn_filters: dictionaries of standard filter ops for standard datatypes @@ -1212,7 +1220,7 @@ class ModelFilterParser(HasAModelManager): return { "op": { "eq": lambda i, v: v == getattr(i, key), - "contains": lambda i, v: v in getattr(i, key), + "contains": lambda i, v: v in partial(getattr(i, key), autoescape=True), } } diff --git a/lib/galaxy/managers/history_contents.py b/lib/galaxy/managers/history_contents.py index a041236328f..34131c07af7 100644 --- a/lib/galaxy/managers/history_contents.py +++ b/lib/galaxy/managers/history_contents.py @@ -331,8 +331,8 @@ class HistoryContentsManager(base.SortableManager): contained_query = contained_query.filter(orm_filter.filter(self.contained_class)) subcontainer_query = subcontainer_query.filter(orm_filter.filter(self.subcontainer_class)) elif orm_filter.filter_type == "orm": - contained_query = self._apply_orm_filter(contained_query, orm_filter.filter) - subcontainer_query = self._apply_orm_filter(subcontainer_query, orm_filter.filter) + contained_query = self._apply_orm_filter(contained_query, orm_filter) + subcontainer_query = self._apply_orm_filter(subcontainer_query, orm_filter) contents_query = contained_query.union_all(subcontainer_query) contents_query = contents_query.order_by(*order_by) @@ -344,10 +344,12 @@ class HistoryContentsManager(base.SortableManager): return contents_query def _apply_orm_filter(self, qry, orm_filter): - if isinstance(orm_filter, sql.elements.BinaryExpression): - for match in filter(lambda col: col["name"] == orm_filter.left.name, qry.column_descriptions): + if isinstance(orm_filter.filter, sql.elements.BinaryExpression): + for match in filter(lambda col: col["name"] == orm_filter.filter.left.name, qry.column_descriptions): column = match["expr"] - new_filter = orm_filter._clone() + new_filter = orm_filter.filter._clone() + if orm_filter.case_insensitive: + column = func.lower(column) new_filter.left = column qry = qry.filter(new_filter) return qry diff --git a/lib/galaxy/tool_util/deps/container_classes.py b/lib/galaxy/tool_util/deps/container_classes.py index f60a18cb94f..a7a242eece3 100644 --- a/lib/galaxy/tool_util/deps/container_classes.py +++ b/lib/galaxy/tool_util/deps/container_classes.py @@ -441,6 +441,7 @@ class SingularityContainer(Container, HasDockerLikeVolumes): run_extra_arguments=self.prop("run_extra_arguments", singularity_util.DEFAULT_RUN_EXTRA_ARGUMENTS), guest_ports=self.tool_info.guest_ports, container_name=self.container_name, + cleanenv=asbool(self.prop("cleanenv", singularity_util.DEFAULT_CLEANENV)), **self.get_singularity_target_kwds(), ) return run_command diff --git a/lib/galaxy/tool_util/deps/requirements.py b/lib/galaxy/tool_util/deps/requirements.py index 5e05dc2fea4..628e245efd4 100644 --- a/lib/galaxy/tool_util/deps/requirements.py +++ b/lib/galaxy/tool_util/deps/requirements.py @@ -1,4 +1,5 @@ import copy +import os from typing import ( Any, Callable, @@ -187,8 +188,14 @@ class ContainerDescription: resolve_dependencies: bool = DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES, shell: str = DEFAULT_CONTAINER_SHELL, ) -> None: - # Force to lowercase because container image names must be lowercase - self.identifier = identifier.lower() if identifier else None + # Force to lowercase because container image names must be lowercase. + # Cached singularity images include the path on disk, so only lowercase + # the image identifier portion. + self.identifier = None + if identifier: + parts = identifier.rsplit(os.sep, 1) + parts[-1] = parts[-1].lower() + self.identifier = os.sep.join(parts) self.type = type self.resolve_dependencies = resolve_dependencies self.shell = shell diff --git a/lib/galaxy/tool_util/deps/singularity_util.py b/lib/galaxy/tool_util/deps/singularity_util.py index c783782c7d6..33d8559cd33 100644 --- a/lib/galaxy/tool_util/deps/singularity_util.py +++ b/lib/galaxy/tool_util/deps/singularity_util.py @@ -3,6 +3,7 @@ import shlex DEFAULT_WORKING_DIRECTORY = None DEFAULT_SINGULARITY_COMMAND = "singularity" +DEFAULT_CLEANENV = True DEFAULT_SUDO = False DEFAULT_SUDO_COMMAND = "sudo" DEFAULT_RUN_EXTRA_ARGUMENTS = None @@ -58,6 +59,7 @@ def build_singularity_run_command( sudo_cmd=DEFAULT_SUDO_COMMAND, guest_ports=False, container_name=None, + cleanenv=DEFAULT_CLEANENV, ): volumes = volumes or [] env = env or [] @@ -73,6 +75,8 @@ def build_singularity_run_command( sudo=sudo, sudo_cmd=sudo_cmd, ) + if cleanenv: + command_parts.append("--cleanenv") command_parts.append("-s") command_parts.append("exec") for volume in volumes: diff --git a/lib/galaxy/util/zipstream.py b/lib/galaxy/util/zipstream.py index 53dd88d5707..5bddbb68ab5 100644 --- a/lib/galaxy/util/zipstream.py +++ b/lib/galaxy/util/zipstream.py @@ -1,10 +1,14 @@ import os +import zlib from urllib.parse import quote import zipstream from .path import safe_walk +CRC32_MIN = 1444 +CRC32_MAX = 1459 + class ZipstreamWrapper: def __init__(self, archive_name=None, upstream_mod_zip=False, upstream_gzip=False): @@ -15,11 +19,13 @@ class ZipstreamWrapper: allowZip64=True, compression=zipstream.ZIP_STORED if upstream_gzip else zipstream.ZIP_DEFLATED ) self.files = [] + self.directories = set() self.size = 0 def response(self): if self.upstream_mod_zip: - yield "\n".join(self.files).encode() + dir_lines = [f"0 0 @directory {directory}" for directory in self.directories] + yield "\n".join(dir_lines + self.files).encode() else: yield from iter(self.archive) @@ -38,7 +44,16 @@ class ZipstreamWrapper: if self.upstream_mod_zip: # calculating crc32 would defeat the point of using mod-zip, but if we ever calculate hashsums we should consider this crc32 = "-" + # We do have to calculate the crc32 for files that are between 1444 and 1459 bytes in size, xref: https://github.com/evanmiller/mod_zip/issues/44#issuecomment-656660686 + # Oddly that seems to be only true for usegalaxy.org (nginx version 1.12.2), and works fine locally (nginx 1.19.10). + # May have been fixed in nginx 1.17.0 + if CRC32_MIN <= os.path.getsize(path) <= CRC32_MAX: + with open(path, "rb") as contents: + crc32 = hex(zlib.crc32(contents.read()))[2:] line = f"{crc32} {size} {quote(path)} {archive_name}" + head, tail = os.path.split(archive_name) + if head: + self.directories.add(head) self.files.append(line) else: self.size += size diff --git a/lib/galaxy_test/api/test_history_contents.py b/lib/galaxy_test/api/test_history_contents.py index 511f58a4046..fc6beed6627 100644 --- a/lib/galaxy_test/api/test_history_contents.py +++ b/lib/galaxy_test/api/test_history_contents.py @@ -838,6 +838,33 @@ class HistoryContentsApiTestCase(ApiTestCase): contents_response = self._get(f"histories/{history_id}/contents?types=dataset&types=dataset_collection").json() assert len(contents_response) == expected_num_datasets + expected_num_collections + def test_index_filter_by_name_ignores_case(self): + history_id = self.dataset_populator.new_history() + self.dataset_populator.new_dataset(history_id, name="AC") + self.dataset_populator.new_dataset(history_id, name="ac") + self.dataset_populator.new_dataset(history_id, name="Bc") + + contains_text = "a" + contents_response = self._get( + f"histories/{history_id}/contents?v=dev&q=name-contains&qv={contains_text}" + ).json() + assert len(contents_response) == 2 + contains_text = "b" + contents_response = self._get( + f"histories/{history_id}/contents?v=dev&q=name-contains&qv={contains_text}" + ).json() + assert len(contents_response) == 1 + contains_text = "c" + contents_response = self._get( + f"histories/{history_id}/contents?v=dev&q=name-contains&qv={contains_text}" + ).json() + assert len(contents_response) == 3 + contains_text = "%" + contents_response = self._get( + f"histories/{history_id}/contents?v=dev&q=name-contains&qv={contains_text}" + ).json() + assert len(contents_response) == 0 + def test_elements_datatypes_field(self): history_id = self.dataset_populator.new_history() collection_name = "homogeneous" diff --git a/test/unit/app/jobs/job_conf.sample_advanced.yml b/test/unit/app/jobs/job_conf.sample_advanced.yml index 811034e6786..5d9a2bcbfc1 100644 --- a/test/unit/app/jobs/job_conf.sample_advanced.yml +++ b/test/unit/app/jobs/job_conf.sample_advanced.yml @@ -16,11 +16,11 @@ runners: load: galaxy.jobs.runners.drmaa:DRMAAJobRunner # Override the $DRMAA_LIBRARY_PATH environment variable drmaa_library_path: /sge/lib/libdrmaa.so - cli: + cli: load: galaxy.jobs.runners.cli:ShellJobRunner - condor: + condor: load: galaxy.jobs.runners.condor:CondorJobRunner - slurm: + slurm: load: galaxy.jobs.runners.slurm:SlurmJobRunner dynamic: # The dynamic runner is not a real job running plugin and is @@ -38,7 +38,7 @@ runners: # The shared file system needs to be exposed to k8s through a Persistent Volume (rw) and a Persistent # Volume Claim. An example of a Persistent Volume could be, in yaml (access modes, reclaim policy and # path are relevant) (persistent_volume.yaml): - # + # # kind: PersistentVolume # apiVersion: v1 # metadata: @@ -217,7 +217,7 @@ runners: # Chronos is a framework for the Apache Mesos software; a software which manages # computer clusters. Specifically, Chronos runs of top of Mesos and it's used # for job orchestration. - # + # # This runner requires a shared file system where the directories of # `job_working_directory`, `file_path` and `new_file_path` settings defined on # the `galaxy.ini` file are shared amongst the Mesos agents (i.e. nodes which @@ -254,7 +254,7 @@ runners: # AMQP does not guarantee that a published message is received by # the AMQP server, so Galaxy/Pulsar can request that the consumer # acknowledge messages and will resend them if acknowledgement is - # not received after a configurable timeout. + # not received after a configurable timeout. #amqp_acknowledge: false # Galaxy reuses Pulsar's persistence_directory parameter (via the @@ -328,7 +328,7 @@ runners: # Specify a complete description of the Pulsar app # to create. If this configuration defines more than - # one manager - you can specify the manager name + # one manager - you can specify the manager name # using the "manager" destination parameter. For more # information on configuring a Pulsar app see: # https://github.com/galaxyproject/pulsar/blob/master/app.yml.sample @@ -483,7 +483,7 @@ execution: # command. #docker_auto_rm: true - # Override which user to launch Docker container as - defaults to + # Override which user to launch Docker container as - defaults to # Galaxy's user id. For remote job execution (e.g. Pulsar) set to # remote job user. Leave empty to not use the -u argument with # Docker. @@ -491,7 +491,7 @@ execution: # Pass extra arguments to the docker run command not covered by the # above options. - #docker_run_extra_arguments: + #docker_run_extra_arguments: # Following command can be used to tweak docker command. #docker_cmd: /usr/local/custom_docker/docker @@ -524,7 +524,7 @@ execution: runner: local # Enable Singularity execution of tools with the follow property. singularity_enabled: true - + # See the above documentation for docker_volumes, singularity_volumes works # almost the same way. The only difference is that $default will expand with # rw directories that in Docker would expand as ro if any of subdirectories are rw. @@ -544,6 +544,11 @@ execution: # default. #singularity_sudo_cmd: /usr/bin/sudo -extra_param + # Singularity by default passes most host environment variables into the container. + # This may be a security or configuration issue, hence galaxy passes the `--cleanenv` + # argument by default. You can turn this off by setting singularity_cleanenv to `false`. + #singularity_cleanenv: true + # Pass extra arguments to the singularity exec command not covered by the # above options. #singularity_run_extra_arguments: '' @@ -776,11 +781,11 @@ execution: runner: pulsar_rest # URL of Pulsar server. url: https://examle.com:8913/ - + # If set, private_token must match token in remote Pulsar's # configuration. private_token: 123456789changeme - + # Uncomment the following statement to disable file staging (e.g. # if there is a shared file system between Galaxy and the Pulsar # server). Alternatively action can be set to 'copy' - to replace @@ -797,7 +802,7 @@ execution: # for more details and examples. #file_action_config: file_actions.yaml #file_actions: {} - + # The non-legacy Pulsar runners will attempt to resolve Galaxy # dependencies remotely - to enable this set a tool_dependency_dir # in Pulsar's configuration (can work with all the same dependency @@ -810,21 +815,21 @@ execution: # Uncomment following option to enable setting metadata on remote # Pulsar server. The 'use_remote_datatypes' option is available for # determining whether to use remotely configured datatypes or local - # ones (both alternatives are a little brittle). + # ones (both alternatives are a little brittle). #remote_metadata: true #use_remote_datatypes: false #remote_property_galaxy_home: /path/to/remote/galaxy-central - + # If remote Pulsar server is configured to run jobs as the real user, # uncomment the following line to pass the current Galaxy user - # along. + # along. #submit_user: $__user_name__ - + # Various other submission parameters can be passed along to the Pulsar # whose use will depend on the remote Pulsar's configured job manager. # For instance: #submit_native_specification: -P bignodes -R y -pe threads 8 - + # Disable parameter rewriting and rewrite generated commands # instead. This may be required if remote host is Windows machine # but probably not otherwise. @@ -926,7 +931,7 @@ execution: # handler will be reused for the resubmitted job. The ``environment`` attribute # is optional, if not present the job's original environment will be reused for the # re-submission. The ``delay`` attribute is optional, if present it will cause the job to - # delay for that number of seconds before being re-submitted. + # delay for that number of seconds before being re-submitted. short_fast: runner: slurm native_specification: '--time=00:05:00 --nodes=1' @@ -979,7 +984,7 @@ execution: # Mount the godocker volumes volumes must be separated by commas. #godocker_volumes: home,galaxy - # If a tool execution in container requires galaxy virtualenv, + # If a tool execution in container requires galaxy virtualenv, # then enable it by setting the value to true. # Disable venv by setting the value to false. virtualenv: false @@ -1002,7 +1007,7 @@ execution: # Tools can be configured to use specific destinations or handlers, # identified by either the "id" or "tags" attribute. If assigned to # a tag, a handler or destination that matches that tag will be -# chosen at random. +# chosen at random. tools: - id: bwa handler: handler0 @@ -1010,23 +1015,23 @@ tools: handler: handler1 - id: bar enviroment: dynamic -- +- # Next example defines resource group to insert into tool interface # and pass to dynamic destination (as resource_params argument). id: longbar environment: dynamic resources: all -- +- # Pick a handler randomly from those declaring this tag. id: baz handler: special_handlers environment: bigmem -- +- # legacy trackerster parameter for tool mapping id: foo handler: handler0 source: trackster -- +- # Classes can be used to map groups of tools - the current classes include # - local (these special tools that aren't parameterized for remote execution - expression tools, upload, etc..) # - requires_galaxy (these special tools require Galaxy's Python environment during execution) @@ -1055,18 +1060,18 @@ resources: # handle jobs for the same environment. To prevent this, assign all # jobs for a specific environment to a single handler. limits: -- +- # Limit on the number of jobs a user with a registered Galaxy # account can have active across all environments. type: registered_user_concurrent_jobs value: 2 -- +- # Likewise, but for unregistered/anonymous users. type: anonymous_user_concurrent_jobs value: 1 -- +- # The number of jobs a user can have active in the specified # environment, or across all environments identified by the # specified tag. (formerly: concurrent_jobs) @@ -1103,7 +1108,7 @@ limits: type: walltime value: '24:00:00' -- +- # Total walltime that jobs may not exceed during a set period. # If total walltime of finished jobs exceeds this value, any # new jobs are paused. `window` is a number in days, @@ -1112,7 +1117,7 @@ limits: window: 30 value: '24:00:00' -- +- # Size that any defined tool output can grow to before the job # will be terminated. This does not include temporary files # created by the job. Format is flexible, e.g.: diff --git a/test/unit/tool_util/test_container_description.py b/test/unit/tool_util/test_container_description.py new file mode 100644 index 00000000000..9251069818d --- /dev/null +++ b/test/unit/tool_util/test_container_description.py @@ -0,0 +1,24 @@ +import pytest + +from galaxy.tool_util.deps.requirements import ContainerDescription + + +@pytest.mark.parametrize( + "identifier,expected_identifier", + [ + ("mulled-abc", "mulled-abc"), + ("docker://mulled-abc", "docker://mulled-abc"), + ("/Cache/mulled-abc", "/Cache/mulled-abc"), + ("/Cache/mUlled-abc", "/Cache/mulled-abc"), + ("", None), + (None, None), + ], +) +def test_container_description(identifier, expected_identifier): + assert ContainerDescription(identifier=identifier).identifier == expected_identifier + + +def test_to_from_dict(): + container_description_dict = ContainerDescription("mulled-abc").to_dict() + container_description = ContainerDescription.from_dict(container_description_dict) + assert container_description_dict == container_description.to_dict()