Merge remote-tracking branch 'upstream/release_22.05' into dev

client/src/components/Workflow/Invocations.test.js

@@ -96,6 +96,9 @@ describe("Invocations.vue", () => {
                 propsData,
                 computed: {
                     getWorkflowNameByInstanceId: (state) => (id) => "workflow name",
+                    getStoredWorkflowIdByInstanceId: (state) => (id) => {
+                        return "workflowId";
+                    },
                     getWorkflowByInstanceId: (state) => (id) => {
                         return { id: "workflowId" };
                     },

client/src/components/Workflow/Invocations.vue

@@ -65,7 +65,10 @@
                 <UtcDate :date="data.value" mode="elapsed" />
             </template>
             <template v-slot:cell(execute)="data">
-                <WorkflowRunButton :id="getWorkflowByInstanceId(data.item.workflow_id).id" :root="root" />
+                <WorkflowRunButton
+                    :id="getStoredWorkflowIdByInstanceId(data.item.workflow_id)"
+                    :root="root"
+                    v-if="getStoredWorkflowIdByInstanceId(data.item.workflow_id)" />
             </template>
         </b-table>
         <b-pagination
@@ -122,7 +125,7 @@ export default {
         };
     },
     computed: {
-        ...mapGetters(["getWorkflowNameByInstanceId", "getWorkflowByInstanceId"]),
+        ...mapGetters(["getWorkflowNameByInstanceId", "getWorkflowByInstanceId", "getStoredWorkflowIdByInstanceId"]),
         ...mapGetters("history", ["getHistoryById", "getHistoryNameById"]),
         title() {
             let title = `Workflow Invocations`;

client/src/store/workflowStore.js

@@ -18,6 +18,10 @@ const getters = {
             return "...";
         }
     },
+    getStoredWorkflowIdByInstanceId: (state) => (workflowId) => {
+        const storedWorkflow = state.workflowsByInstanceId[workflowId];
+        return storedWorkflow?.id;
+    },
 };
 
 const actions = {

doc/source/admin/nginx.md

@@ -442,7 +442,7 @@ galaxy:
 Galaxy creates zip archives when downloading multiple datasets from a history or a dataset library.
 While this works fine for small datasets and few users, nginx can handle the creation of zip archives
 more efficiently using [mod-zip](https://www.nginx.com/resources/wiki/modules/zip/).
-To use this feature, install nginx with mod-zip enabled, provide the file locations from which
+To use this feature, install nginx with mod-zip enabled (requires <https://github.com/evanmiller/mod_zip/commit/51cf45d3e9f51e02224af017b235d1d30fbf28fb> or a newer), provide the file locations from which
 nginx should serve files and edit `galaxy.yml` and make the following changes before restarting Galaxy:
 
 ```yaml

lib/galaxy/config/sample/job_conf.xml.sample_advanced

@@ -606,6 +606,11 @@
           <!-- Following option can be used to tweak sudo command used by
                default. -->
           <!-- <param id="singularity_sudo_cmd">/usr/bin/sudo -extra_param</param> -->
+          <!-- Singularity by default passes most host environment variables into the container.
+               This may be a security or configuration issue, hence galaxy passes the `cleanenv`
+               argument by default. You can turn this off by setting singularity_cleanenv to `false`.
+          -->
+          <!-- <param id="singularity_cleanenv">true</param> -->
           <!-- Pass extra arguments to the singularity exec command not covered by the
                above options. -->
           <!-- <param id="singularity_run_extra_arguments"></param> -->

lib/galaxy/datatypes/goldenpath.py

@@ -1,4 +1,5 @@
 import abc
+import logging
 import os
 from typing import (
     Set,
@@ -12,6 +13,8 @@ from galaxy.datatypes.sniff import (
 )
 from .tabular import Tabular
 
+log = logging.getLogger(__name__)
+
 
 @build_sniff_from_prefix
 class GoldenPath(Tabular):
@@ -600,4 +603,6 @@ class AGPGapLine(AGPLine):
                 )
         else:
             if "na" in all_evidence:
-                raise AGPError(self.fname, self.line_number, "'na' is invalid linkage evidence when asserting linkage")
+                log.warning(
+                    AGPError(self.fname, self.line_number, "'na' is invalid linkage evidence when asserting linkage")
+                )

lib/galaxy/datatypes/test/eg2.agp

@@ -53,4 +53,4 @@ chrY	11824573	11874572	43	N	50000	contig	no	na
 chrY	11874573	11878623	44	W	EG1_scaffold22	1	4051	+
 chrY	11878624	11928623	45	N	50000	contig	no	na
 chrY	11928624	11933053	46	W	EG1_scaffold23	1	4430	+
-chrY	11933054	11943053	47	N	10000	telomere	no	na
+chrY	11933054	11943053	47	N	10000	telomere	yes	na

lib/galaxy/managers/base.py

@@ -28,6 +28,7 @@ attribute change to a model object.
 import datetime
 import logging
 import re
+from functools import partial
 from typing import (
     Any,
     Callable,
@@ -68,6 +69,7 @@ log = logging.getLogger(__name__)
 class ParsedFilter(NamedTuple):
     filter_type: str  # orm_function, function, or orm
     filter: Any
+    case_insensitive: bool = False
 
 
 parsed_filter = ParsedFilter
@@ -1169,6 +1171,7 @@ class ModelFilterParser(HasAModelManager):
         allowed_ops = column_map["op"]
         if op not in allowed_ops:
             return None
+
         converted_op = self._convert_op_string_to_fn(column, op)
         if not converted_op:
             return None
@@ -1181,9 +1184,12 @@ class ModelFilterParser(HasAModelManager):
             val_parser = val_parser.get(op)
         if val_parser:
             val = val_parser(val)
+        if op == "contains":
+            # Do we want to make this configurable ?
+            val = val.lower()
 
         orm_filter = converted_op(val)
-        return self.parsed_filter(filter_type="orm", filter=orm_filter)
+        return self.parsed_filter(filter_type="orm", filter=orm_filter, case_insensitive=op == "contains")
 
     #: these are the easier/shorter string equivalents to the python operator fn names that need '__' around them
     UNDERSCORED_OPS = ("lt", "le", "eq", "ne", "ge", "gt")
@@ -1205,6 +1211,8 @@ class ModelFilterParser(HasAModelManager):
         op_fn = getattr(column, fn_name, None)
         if not op_fn or not callable(op_fn):
             return None
+        if op_string == "contains":
+            op_fn = partial(op_fn, autoescape=True)
         return op_fn
 
     # ---- preset fn_filters: dictionaries of standard filter ops for standard datatypes
@@ -1212,7 +1220,7 @@ class ModelFilterParser(HasAModelManager):
         return {
             "op": {
                 "eq": lambda i, v: v == getattr(i, key),
-                "contains": lambda i, v: v in getattr(i, key),
+                "contains": lambda i, v: v in partial(getattr(i, key), autoescape=True),
             }
         }
 

lib/galaxy/managers/history_contents.py

@@ -331,8 +331,8 @@ class HistoryContentsManager(base.SortableManager):
                 contained_query = contained_query.filter(orm_filter.filter(self.contained_class))
                 subcontainer_query = subcontainer_query.filter(orm_filter.filter(self.subcontainer_class))
             elif orm_filter.filter_type == "orm":
-                contained_query = self._apply_orm_filter(contained_query, orm_filter.filter)
-                subcontainer_query = self._apply_orm_filter(subcontainer_query, orm_filter.filter)
+                contained_query = self._apply_orm_filter(contained_query, orm_filter)
+                subcontainer_query = self._apply_orm_filter(subcontainer_query, orm_filter)
 
         contents_query = contained_query.union_all(subcontainer_query)
         contents_query = contents_query.order_by(*order_by)
@@ -344,10 +344,12 @@ class HistoryContentsManager(base.SortableManager):
         return contents_query
 
     def _apply_orm_filter(self, qry, orm_filter):
-        if isinstance(orm_filter, sql.elements.BinaryExpression):
-            for match in filter(lambda col: col["name"] == orm_filter.left.name, qry.column_descriptions):
+        if isinstance(orm_filter.filter, sql.elements.BinaryExpression):
+            for match in filter(lambda col: col["name"] == orm_filter.filter.left.name, qry.column_descriptions):
                 column = match["expr"]
-                new_filter = orm_filter._clone()
+                new_filter = orm_filter.filter._clone()
+                if orm_filter.case_insensitive:
+                    column = func.lower(column)
                 new_filter.left = column
                 qry = qry.filter(new_filter)
         return qry

lib/galaxy/tool_util/deps/container_classes.py

@@ -441,6 +441,7 @@ class SingularityContainer(Container, HasDockerLikeVolumes):
             run_extra_arguments=self.prop("run_extra_arguments", singularity_util.DEFAULT_RUN_EXTRA_ARGUMENTS),
             guest_ports=self.tool_info.guest_ports,
             container_name=self.container_name,
+            cleanenv=asbool(self.prop("cleanenv", singularity_util.DEFAULT_CLEANENV)),
             **self.get_singularity_target_kwds(),
         )
         return run_command

lib/galaxy/tool_util/deps/requirements.py

@@ -1,4 +1,5 @@
 import copy
+import os
 from typing import (
     Any,
     Callable,
@@ -187,8 +188,14 @@ class ContainerDescription:
         resolve_dependencies: bool = DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES,
         shell: str = DEFAULT_CONTAINER_SHELL,
     ) -> None:
-        # Force to lowercase because container image names must be lowercase
-        self.identifier = identifier.lower() if identifier else None
+        # Force to lowercase because container image names must be lowercase.
+        # Cached singularity images include the path on disk, so only lowercase
+        # the image identifier portion.
+        self.identifier = None
+        if identifier:
+            parts = identifier.rsplit(os.sep, 1)
+            parts[-1] = parts[-1].lower()
+            self.identifier = os.sep.join(parts)
         self.type = type
         self.resolve_dependencies = resolve_dependencies
         self.shell = shell

lib/galaxy/tool_util/deps/singularity_util.py

@@ -3,6 +3,7 @@ import shlex
 
 DEFAULT_WORKING_DIRECTORY = None
 DEFAULT_SINGULARITY_COMMAND = "singularity"
+DEFAULT_CLEANENV = True
 DEFAULT_SUDO = False
 DEFAULT_SUDO_COMMAND = "sudo"
 DEFAULT_RUN_EXTRA_ARGUMENTS = None
@@ -58,6 +59,7 @@ def build_singularity_run_command(
     sudo_cmd=DEFAULT_SUDO_COMMAND,
     guest_ports=False,
     container_name=None,
+    cleanenv=DEFAULT_CLEANENV,
 ):
     volumes = volumes or []
     env = env or []
@@ -73,6 +75,8 @@ def build_singularity_run_command(
         sudo=sudo,
         sudo_cmd=sudo_cmd,
     )
+    if cleanenv:
+        command_parts.append("--cleanenv")
     command_parts.append("-s")
     command_parts.append("exec")
     for volume in volumes:

lib/galaxy/util/zipstream.py

@@ -1,10 +1,14 @@
 import os
+import zlib
 from urllib.parse import quote
 
 import zipstream
 
 from .path import safe_walk
 
+CRC32_MIN = 1444
+CRC32_MAX = 1459
+
 
 class ZipstreamWrapper:
     def __init__(self, archive_name=None, upstream_mod_zip=False, upstream_gzip=False):
@@ -15,11 +19,13 @@ class ZipstreamWrapper:
                 allowZip64=True, compression=zipstream.ZIP_STORED if upstream_gzip else zipstream.ZIP_DEFLATED
             )
         self.files = []
+        self.directories = set()
         self.size = 0
 
     def response(self):
         if self.upstream_mod_zip:
-            yield "\n".join(self.files).encode()
+            dir_lines = [f"0 0 @directory {directory}" for directory in self.directories]
+            yield "\n".join(dir_lines + self.files).encode()
         else:
             yield from iter(self.archive)
 
@@ -38,7 +44,16 @@ class ZipstreamWrapper:
         if self.upstream_mod_zip:
             # calculating crc32 would defeat the point of using mod-zip, but if we ever calculate hashsums we should consider this
             crc32 = "-"
+            # We do have to calculate the crc32 for files that are between 1444 and 1459 bytes in size, xref: https://github.com/evanmiller/mod_zip/issues/44#issuecomment-656660686
+            # Oddly that seems to be only true for usegalaxy.org (nginx version 1.12.2), and works fine locally (nginx 1.19.10).
+            # May have been fixed in nginx 1.17.0
+            if CRC32_MIN <= os.path.getsize(path) <= CRC32_MAX:
+                with open(path, "rb") as contents:
+                    crc32 = hex(zlib.crc32(contents.read()))[2:]
             line = f"{crc32} {size} {quote(path)} {archive_name}"
+            head, tail = os.path.split(archive_name)
+            if head:
+                self.directories.add(head)
             self.files.append(line)
         else:
             self.size += size

lib/galaxy_test/api/test_history_contents.py

@@ -838,6 +838,33 @@ class HistoryContentsApiTestCase(ApiTestCase):
         contents_response = self._get(f"histories/{history_id}/contents?types=dataset&types=dataset_collection").json()
         assert len(contents_response) == expected_num_datasets + expected_num_collections
 
+    def test_index_filter_by_name_ignores_case(self):
+        history_id = self.dataset_populator.new_history()
+        self.dataset_populator.new_dataset(history_id, name="AC")
+        self.dataset_populator.new_dataset(history_id, name="ac")
+        self.dataset_populator.new_dataset(history_id, name="Bc")
+
+        contains_text = "a"
+        contents_response = self._get(
+            f"histories/{history_id}/contents?v=dev&q=name-contains&qv={contains_text}"
+        ).json()
+        assert len(contents_response) == 2
+        contains_text = "b"
+        contents_response = self._get(
+            f"histories/{history_id}/contents?v=dev&q=name-contains&qv={contains_text}"
+        ).json()
+        assert len(contents_response) == 1
+        contains_text = "c"
+        contents_response = self._get(
+            f"histories/{history_id}/contents?v=dev&q=name-contains&qv={contains_text}"
+        ).json()
+        assert len(contents_response) == 3
+        contains_text = "%"
+        contents_response = self._get(
+            f"histories/{history_id}/contents?v=dev&q=name-contains&qv={contains_text}"
+        ).json()
+        assert len(contents_response) == 0
+
     def test_elements_datatypes_field(self):
         history_id = self.dataset_populator.new_history()
         collection_name = "homogeneous"

test/unit/app/jobs/job_conf.sample_advanced.yml

@@ -16,11 +16,11 @@ runners:
     load: galaxy.jobs.runners.drmaa:DRMAAJobRunner
     # Override the $DRMAA_LIBRARY_PATH environment variable
     drmaa_library_path: /sge/lib/libdrmaa.so
-  cli: 
+  cli:
     load: galaxy.jobs.runners.cli:ShellJobRunner
-  condor: 
+  condor:
     load: galaxy.jobs.runners.condor:CondorJobRunner
-  slurm: 
+  slurm:
     load: galaxy.jobs.runners.slurm:SlurmJobRunner
   dynamic:
     # The dynamic runner is not a real job running plugin and is
@@ -38,7 +38,7 @@ runners:
     # The shared file system needs to be exposed to k8s through a Persistent Volume (rw) and a Persistent
     # Volume Claim. An example of a Persistent Volume could be, in yaml (access modes, reclaim policy and
     # path are relevant) (persistent_volume.yaml):
-    # 
+    #
     #     kind: PersistentVolume
     #     apiVersion: v1
     #     metadata:
@@ -217,7 +217,7 @@ runners:
     # Chronos is a framework for the Apache Mesos software; a software which manages
     # computer clusters. Specifically, Chronos runs of top of Mesos and it's used
     # for job orchestration.
-    # 
+    #
     # This runner requires a shared file system where the directories of
     # `job_working_directory`, `file_path` and `new_file_path` settings defined on
     # the `galaxy.ini` file are shared amongst the Mesos agents (i.e. nodes which
@@ -254,7 +254,7 @@ runners:
     # AMQP does not guarantee that a published message is received by
     # the AMQP server, so Galaxy/Pulsar can request that the consumer
     # acknowledge messages and will resend them if acknowledgement is
-    # not received after a configurable timeout. 
+    # not received after a configurable timeout.
     #amqp_acknowledge: false
 
     # Galaxy reuses Pulsar's persistence_directory parameter (via the
@@ -328,7 +328,7 @@ runners:
 
     # Specify a complete description of the Pulsar app
     # to create. If this configuration defines more than
-    # one manager - you can specify the manager name 
+    # one manager - you can specify the manager name
     # using the "manager" destination parameter. For more
     # information on configuring a Pulsar app see:
     #  https://github.com/galaxyproject/pulsar/blob/master/app.yml.sample
@@ -483,7 +483,7 @@ execution:
       # command.
       #docker_auto_rm: true
 
-      # Override which user to launch Docker container as - defaults to 
+      # Override which user to launch Docker container as - defaults to
       # Galaxy's user id. For remote job execution (e.g. Pulsar) set to
       # remote job user. Leave empty to not use the -u argument with
       # Docker.
@@ -491,7 +491,7 @@ execution:
 
       # Pass extra arguments to the docker run command not covered by the
       # above options.
-      #docker_run_extra_arguments: 
+      #docker_run_extra_arguments:
 
       # Following command can be used to tweak docker command.
       #docker_cmd: /usr/local/custom_docker/docker
@@ -524,7 +524,7 @@ execution:
       runner: local
       # Enable Singularity execution of tools with the follow property.
       singularity_enabled: true
-      
+
       # See the above documentation for docker_volumes, singularity_volumes works
       # almost the same way. The only difference is that $default will expand with
       # rw directories that in Docker would expand as ro if any of subdirectories are rw.
@@ -544,6 +544,11 @@ execution:
       # default.
       #singularity_sudo_cmd: /usr/bin/sudo -extra_param
 
+      # Singularity by default passes most host environment variables into the container.
+      # This may be a security or configuration issue, hence galaxy passes the `--cleanenv`
+      # argument by default. You can turn this off by setting singularity_cleanenv to `false`.
+      #singularity_cleanenv: true
+
       # Pass extra arguments to the singularity exec command not covered by the
       # above options.
       #singularity_run_extra_arguments: ''
@@ -776,11 +781,11 @@ execution:
        runner: pulsar_rest
        # URL of Pulsar server.
        url: https://examle.com:8913/
-       
+
        # If set, private_token must match token in remote Pulsar's
        # configuration.
        private_token: 123456789changeme
-       
+
        # Uncomment the following statement to disable file staging (e.g.
        # if there is a shared file system between Galaxy and the Pulsar
        # server). Alternatively action can be set to 'copy' - to replace
@@ -797,7 +802,7 @@ execution:
        # for more details and examples.
        #file_action_config: file_actions.yaml
        #file_actions: {}
-       
+
        # The non-legacy Pulsar runners will attempt to resolve Galaxy
        # dependencies remotely - to enable this set a tool_dependency_dir
        # in Pulsar's configuration (can work with all the same dependency
@@ -810,21 +815,21 @@ execution:
        # Uncomment following option to enable setting metadata on remote
        # Pulsar server. The 'use_remote_datatypes' option is available for
        # determining whether to use remotely configured datatypes or local
-       # ones (both alternatives are a little brittle). 
+       # ones (both alternatives are a little brittle).
        #remote_metadata: true
        #use_remote_datatypes: false
        #remote_property_galaxy_home: /path/to/remote/galaxy-central
-       
+
        # If remote Pulsar server is configured to run jobs as the real user,
        # uncomment the following line to pass the current Galaxy user
-       # along. 
+       # along.
        #submit_user: $__user_name__
-       
+
        # Various other submission parameters can be passed along to the Pulsar
        # whose use will depend on the remote Pulsar's configured job manager.
        # For instance:
        #submit_native_specification: -P bignodes -R y -pe threads 8
-       
+
        # Disable parameter rewriting and rewrite generated commands
        # instead. This may be required if remote host is Windows machine
        # but probably not otherwise.
@@ -926,7 +931,7 @@ execution:
     # handler will be reused for the resubmitted job. The ``environment`` attribute
     # is optional, if not present the job's original environment will be reused for the
     # re-submission. The ``delay`` attribute is optional, if present it will cause the job to
-    # delay for that number of seconds before being re-submitted. 
+    # delay for that number of seconds before being re-submitted.
     short_fast:
       runner: slurm
       native_specification: '--time=00:05:00 --nodes=1'
@@ -979,7 +984,7 @@ execution:
       # Mount the godocker volumes volumes must be separated by commas.
       #godocker_volumes: home,galaxy
 
-      # If a tool execution in container requires galaxy virtualenv, 
+      # If a tool execution in container requires galaxy virtualenv,
       # then enable it by setting the value to true.
       # Disable venv by setting the value to false.
       virtualenv: false
@@ -1002,7 +1007,7 @@ execution:
 # Tools can be configured to use specific destinations or handlers,
 # identified by either the "id" or "tags" attribute.  If assigned to
 # a tag, a handler or destination that matches that tag will be
-# chosen at random. 
+# chosen at random.
 tools:
 - id: bwa
   handler: handler0
@@ -1010,23 +1015,23 @@ tools:
   handler: handler1
 - id: bar
   enviroment: dynamic
-- 
+-
   # Next example defines resource group to insert into tool interface
   # and pass to dynamic destination (as resource_params argument).
   id: longbar
   environment: dynamic
   resources: all
-- 
+-
   # Pick a handler randomly from those declaring this tag.
   id: baz
   handler: special_handlers
   environment: bigmem
-- 
+-
   # legacy trackerster parameter for tool mapping
   id: foo
   handler: handler0
   source: trackster
-- 
+-
   # Classes can be used to map groups of tools - the current classes include
   # - local (these special tools that aren't parameterized for remote execution - expression tools, upload, etc..)
   # - requires_galaxy (these special tools require Galaxy's Python environment during execution)
@@ -1055,18 +1060,18 @@ resources:
 # handle jobs for the same environment. To prevent this, assign all
 # jobs for a specific environment to a single handler.
 limits:
-- 
+-
   # Limit on the number of jobs a user with a registered Galaxy
   # account can have active across all environments.
   type: registered_user_concurrent_jobs
   value: 2
 
-- 
+-
   # Likewise, but for unregistered/anonymous users.
   type: anonymous_user_concurrent_jobs
   value: 1
 
-- 
+-
   # The number of jobs a user can have active in the specified
   # environment, or across all environments identified by the
   # specified tag. (formerly: concurrent_jobs)
@@ -1103,7 +1108,7 @@ limits:
   type: walltime
   value: '24:00:00'
 
-- 
+-
   # Total walltime that jobs may not exceed during a set period.
   # If total walltime of finished jobs exceeds this value, any
   # new jobs are paused.  `window` is a number in days,
@@ -1112,7 +1117,7 @@ limits:
   window: 30
   value: '24:00:00'
 
-- 
+-
   # Size that any defined tool output can grow to before the job
   # will be terminated. This does not include temporary files
   # created by the job. Format is flexible, e.g.:

test/unit/tool_util/test_container_description.py

@@ -0,0 +1,24 @@
+import pytest
+
+from galaxy.tool_util.deps.requirements import ContainerDescription
+
+
+@pytest.mark.parametrize(
+    "identifier,expected_identifier",
+    [
+        ("mulled-abc", "mulled-abc"),
+        ("docker://mulled-abc", "docker://mulled-abc"),
+        ("/Cache/mulled-abc", "/Cache/mulled-abc"),
+        ("/Cache/mUlled-abc", "/Cache/mulled-abc"),
+        ("", None),
+        (None, None),
+    ],
+)
+def test_container_description(identifier, expected_identifier):
+    assert ContainerDescription(identifier=identifier).identifier == expected_identifier
+
+
+def test_to_from_dict():
+    container_description_dict = ContainerDescription("mulled-abc").to_dict()
+    container_description = ContainerDescription.from_dict(container_description_dict)
+    assert container_description_dict == container_description.to_dict()