Merge remote-tracking branch 'upstream/release_23.0' into dev

client/src/components/Collections/common/CollectionEditView.vue

@@ -2,15 +2,15 @@
     <div aria-labelledby="collection-edit-view-heading">
         <h1 id="collection-edit-view-heading" class="h-lg">{{ l("Edit Collection Attributes") }}</h1>
         <b-alert show variant="info" dismissible>
-            {{ l(newCollectionInfoMessage) }}
+            {{ l(infoMessage) }}
         </b-alert>
         <div v-if="jobError">
             <b-alert show variant="danger" dismissible>
-                {{ errorMessage }}
+                {{ l(errorMessage) }}
             </b-alert>
         </div>
         <b-tabs content-class="mt-3">
-            <b-tab @click="noQuotaIncrease = true">
+            <b-tab @click="updateInfoMessage(newCollectionMessage + ' ' + noQuotaIncreaseMessage)">
                 <template v-slot:title> <font-awesome-icon icon="table" /> &nbsp; {{ l("Database/Build") }}</template>
                 <db-key-provider v-slot="{ item, loading }">
                     <div v-if="loading"><b-spinner label="Loading Database/Builds..."></b-spinner></div>
@@ -24,13 +24,13 @@
                 </db-key-provider>
             </b-tab>
             <SuitableConvertersProvider :id="collection_id" v-slot="{ item }">
-                <b-tab v-if="item && item.length" @click="noQuotaIncrease = false">
+                <b-tab v-if="item && item.length" @click="updateInfoMessage(newCollectionMessage)">
                     <template v-slot:title> <font-awesome-icon icon="cog" /> &nbsp; {{ l("Convert") }}</template>
                     <suitable-converters-tab :suitable-converters="item" @clicked-convert="clickedConvert" />
                 </b-tab>
             </SuitableConvertersProvider>
             <ConfigProvider v-slot="{ config }">
-                <b-tab v-if="config.enable_celery_tasks">
+                <b-tab v-if="config.enable_celery_tasks" @click="updateInfoMessage(expectWaitTimeMessage)">
                     <template v-slot:title>
                         <font-awesome-icon icon="database" /> &nbsp; {{ l("Datatypes") }}
                     </template>
@@ -94,6 +94,10 @@ export default {
             jobError: null,
             noQuotaIncrease: true,
             loadingString: "Loading Datatypes",
+            infoMessage: "This will create a new collection in your History. Your quota will not increase.", //initialmessage on first/database tab
+            newCollectionMessage: "This will create a new collection in your History.",
+            noQuotaIncreaseMessage: "Your quota will not increase.",
+            expectWaitTimeMessage: "This operation might take a short while, depending on the size of your collection.",
         };
     },
     computed: {
@@ -103,13 +107,6 @@ export default {
         datatypeFromElements: function () {
             return this.attributesData.extension;
         },
-        newCollectionInfoMessage: function () {
-            let newCollectionMessage = "This will create a new collection in your History.";
-            if (this.noQuotaIncrease) {
-                newCollectionMessage += " Your quota usage will not increase.";
-            }
-            return newCollectionMessage;
-        },
         historyId: function () {
             return this.$store.getters["history/currentHistoryId"];
         },
@@ -118,6 +115,9 @@ export default {
         this.getCollectionDataAndAttributes();
     },
     methods: {
+        updateInfoMessage: function (strMessage) {
+            this.infoMessage = strMessage;
+        },
         getCollectionDataAndAttributes: async function () {
             let attributesGet = this.$store.getters.getCollectionAttributes(this.collection_id);
             if (attributesGet == null) {

client/src/components/Grid/GridHistory.vue

@@ -5,8 +5,16 @@
 <script>
 import HistoryList from "./history-list";
 export default {
+    props: {
+        actionId: {
+            type: String,
+            required: true,
+        },
+    },
     mounted() {
-        new HistoryList.View().$el.appendTo(this.$refs.target);
+        new HistoryList.View({
+            action_id: this.actionId,
+        }).$el.appendTo(this.$refs.target);
     },
 };
 </script>

client/src/components/Grid/history-list.js

@@ -65,7 +65,7 @@ var View = Backbone.View.extend({
         this.active_tab = "user";
         this.model = new Backbone.Model();
         Utils.get({
-            url: `${getAppRoot()}history/list?${$.param(Galaxy.params)}`,
+            url: `${getAppRoot()}history/${options.action_id}?${$.param(Galaxy.params)}`,
             success: (response) => {
                 this.model.set(response);
                 this.render();

client/src/components/History/CurrentHistory/HistoryFilters/HistoryFilters.test.js

@@ -38,6 +38,8 @@ describe("HistoryFilters", () => {
             "[placeholder='any extension']": "ext-filter",
             "[placeholder='any tag']": "tag filter",
             "[placeholder='any state']": "state-filter",
+            "[placeholder='any database']": "db-filter",
+            "[placeholder='index equals']": "hid-related",
             "[placeholder='index greater']": "hid-greater",
             "[placeholder='index lower']": "hid-lower",
             "[placeholder='created after']": "January 1, 2022",
@@ -79,7 +81,7 @@ describe("HistoryFilters", () => {
         await expectCorrectEmits(
             wrapper,
             false,
-            "create_time>'January 1, 2022' create_time<'January 1, 2023' extension:ext-filter hid>hid-greater hid<hid-lower name:name-filter state:state-filter tag:'tag filter'"
+            "create_time>'January 1, 2022' create_time<'January 1, 2023' extension:ext-filter genome_build:db-filter related:hid-related hid>hid-greater hid<hid-lower name:name-filter state:state-filter tag:'tag filter'"
         );
 
         // -------- Test esc key:  ---------

client/src/components/History/CurrentHistory/HistoryFilters/HistoryFilters.vue

@@ -46,6 +46,8 @@
             <small class="mt-1">Filter by state:</small>
             <b-form-input v-model="filterSettings['state:']" size="sm" placeholder="any state" list="stateSelect" />
             <b-form-datalist id="stateSelect" :options="states"></b-form-datalist>
+            <small>Filter by database:</small>
+            <b-form-input v-model="filterSettings['genome_build:']" size="sm" placeholder="any database" />
             <small class="mt-1">Filter by related to item index:</small>
             <b-form-input v-model="filterSettings['related:']" size="sm" placeholder="index equals" />
             <small class="mt-1">Filter by item index:</small>

client/src/components/History/HistoryFilters.js

@@ -5,6 +5,7 @@ export const validFilters = {
     state: equals("state"),
     name: contains("name"),
     extension: equals("extension"),
+    genome_build: contains("genome_build"),
     hid_ge: compare("hid", "ge"),
     hid_gt: compare("hid", "gt"),
     hid_le: compare("hid", "le"),

client/src/entry/analysis/router.js

@@ -237,7 +237,7 @@ export function getRouter(Galaxy) {
                         props: true,
                     },
                     {
-                        path: "histories/list",
+                        path: "histories/:actionId",
                         component: GridHistory,
                         props: true,
                         redirect: redirectAnon(),

client/src/utils/navigation/navigation.yml

@@ -309,6 +309,7 @@ edit_dataset_attributes:
 
 edit_collection_attributes:
   selectors:
+    alert_info: 'div.alert-info'
     database_genome_tab:
       type: xpath
       selector: '//a[contains(text(), "Database/Build")]'

client/src/utils/webhooks.js

@@ -29,11 +29,9 @@ const WebhookView = Backbone.View.extend({
         this.$el.attr("tool_version", toolVersion);
 
         getWebHookData().then((data) => {
-            if (options.type) {
-                data.reset(filterType(data, options.type));
-            }
-            if (data.length > 0) {
-                this.render(weightedRandomPick(data));
+            const filteredData = filterData(data, options);
+            if (filteredData.length > 0) {
+                this.render(weightedRandomPick(filteredData));
             }
         });
     },
@@ -46,13 +44,17 @@ const WebhookView = Backbone.View.extend({
     },
 });
 
+function filterData(data, options) {
+    let filteredData = data;
+    if (options.type) {
+        filteredData = filterType(data, options.type);
+    }
+    return filteredData;
+}
+
 const load = (options) => {
     getWebHookData().then((data) => {
-        let filteredData = data;
-        if (options.type) {
-            filteredData = filterType(data, options.type);
-        }
-        options.callback(filteredData);
+        options.callback(filterData(data, options));
     });
 };
 

doc/schema_template.md

@@ -74,6 +74,7 @@ $tag:tool|outputs|data|actions://complexType[@name='Actions']
 $tag:tool|outputs|data|actions|conditional://complexType[@name='ActionsConditional']
 $tag:tool|outputs|data|actions|conditional|when://complexType[@name='ActionsConditionalWhen']
 $tag:tool|outputs|data|actions|action://complexType[@name='Action']
+$tag:tool|outputs|data|actions|action|option://complexType[@name='ActionsOption']
 $tag:tool|outputs|data|discover_datasets://complexType[@name='OutputDiscoverDatasets']
 $tag:tool|outputs|collection://complexType[@name='OutputCollection']
 $tag:tool|outputs|collection|filter://complexType[@name='OutputFilter']

lib/galaxy/managers/genomes.py

@@ -1,8 +1,11 @@
 from typing import (
     Any,
     List,
+    TYPE_CHECKING,
 )
 
+from sqlalchemy import func
+
 from galaxy import model as m
 from galaxy.exceptions import (
     ReferenceDataError,
@@ -10,6 +13,10 @@ from galaxy.exceptions import (
 )
 from galaxy.managers.context import ProvidesUserContext
 from galaxy.structured_app import StructuredApp
+from .base import raise_filter_err
+
+if TYPE_CHECKING:
+    from galaxy.managers.base import OrmFilterParsersType
 
 
 class GenomesManager:
@@ -64,3 +71,30 @@ class GenomesManager:
             raise ReferenceDataError(f"Data tables not found for {index_type} for {id}")
         else:
             return f"{file_name}{ext}"
+
+
+class GenomeFilterMixin:
+    orm_filter_parsers: "OrmFilterParsersType"
+    valid_ops = ("eq", "contains", "has")
+
+    def create_genome_filter(self, attr, op, val):
+        def _create_genome_filter(model_class=None):
+            if op not in GenomeFilterMixin.valid_ops:
+                raise_filter_err(attr, op, val, "bad op in filter")
+            if model_class is None:
+                return True
+            # Doesn't filter genome_build for collections
+            if model_class.__name__ == "HistoryDatasetCollectionAssociation":
+                return False
+            column = func.json_extract(model_class.table.c._metadata, "$.dbkey")
+            lower_val = val.lower()  # Ignore case
+            if op == "eq":
+                cond = func.lower(column) == lower_val
+            else:
+                cond = func.lower(column).contains(lower_val, autoescape=True)
+            return cond
+
+        return _create_genome_filter
+
+    def _add_parsers(self):
+        self.orm_filter_parsers.update({"genome_build": self.create_genome_filter})

lib/galaxy/managers/hdas.py

@@ -337,6 +337,7 @@ class HDASerializer(  # datasets._UnflattenedMetadataDatasetAssociationSerialize
                 "hid",
                 "history_content_type",
                 "dataset_id",
+                "genome_build",
                 "state",
                 "extension",
                 "deleted",

lib/galaxy/managers/history_contents.py

@@ -37,6 +37,7 @@ from galaxy.managers import (
     annotatable,
     base,
     deletable,
+    genomes,
     hdas,
     hdcas,
     taggable,
@@ -521,6 +522,7 @@ class HistoryContentsFilters(
     base.ModelFilterParser,
     annotatable.AnnotatableFilterMixin,
     deletable.PurgableFiltersMixin,
+    genomes.GenomeFilterMixin,
     taggable.TaggableFilterMixin,
     tools.ToolFilterMixin,
 ):
@@ -563,8 +565,10 @@ class HistoryContentsFilters(
                     return sql.column("history_content_type") == val
                 raise_filter_err(attr, op, val, "bad op in filter")
 
-            if attr == "related" and op == "eq":
-                return sql.column("hid").in_(json.loads(val))
+            if attr == "related":
+                if op == "eq":
+                    return sql.column("hid").in_(json.loads(val))
+                raise_filter_err(attr, op, val, "bad op in filter")
 
             if attr == "type_id":
                 if op == "eq":
@@ -617,12 +621,14 @@ class HistoryContentsFilters(
     def _add_parsers(self):
         super()._add_parsers()
         annotatable.AnnotatableFilterMixin._add_parsers(self)
+        genomes.GenomeFilterMixin._add_parsers(self)
         deletable.PurgableFiltersMixin._add_parsers(self)
         taggable.TaggableFilterMixin._add_parsers(self)
         tools.ToolFilterMixin._add_parsers(self)
         self.orm_filter_parsers.update(
             {
                 "history_content_type": {"op": ("eq")},
+                # maybe remove related from here, as there's no corresponding field?
                 "related": {"op": ("eq")},
                 "type_id": {"op": ("eq", "in"), "val": self.parse_type_id_list},
                 "hid": {"op": ("eq", "ge", "le", "gt", "lt"), "val": int},

lib/galaxy/model/store/discover.py

@@ -104,7 +104,6 @@ class ModelPersistenceContext(metaclass=abc.ABCMeta):
 
         if primary_data is not None:
             primary_data.extension = ext
-            primary_data.visible = visible
             primary_data.dbkey = dbkey
         else:
             if not library_folder:
@@ -811,12 +810,13 @@ def persist_hdas(elements, model_persistence_context, final_job_state="ok"):
                 hashes = fields_match.hashes
                 created_from_basename = fields_match.created_from_basename
                 extra_files = fields_match.extra_files
+                visible = fields_match.visible
 
                 info, state = discovered_file.discovered_state(element, final_job_state)
                 dataset = model_persistence_context.create_dataset(
                     ext=ext,
                     designation=designation,
-                    visible=True,
+                    visible=visible,
                     dbkey=dbkey,
                     name=name,
                     filename=discovered_file.path,

lib/galaxy/tool_util/xsd/galaxy.xsd

@@ -5752,22 +5752,30 @@ Source of the tabular data ``from_data_table``, ``from_param``, or ``from_file``
     </xs:attribute>
     <xs:attribute name="name" type="xs:string">
       <xs:annotation>
-        <xs:documentation xml:lang="en"></xs:documentation>
+        <xs:documentation xml:lang="en"><![CDATA[
+Name of the referred data table, parameter, or file (required).
+        ]]></xs:documentation>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="column" type="xs:integer">
       <xs:annotation>
-        <xs:documentation xml:lang="en"></xs:documentation>
+        <xs:documentation xml:lang="en"><![CDATA[
+The column to choose the value from (required)
+        ]]></xs:documentation>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="offset" type="xs:integer">
       <xs:annotation>
-        <xs:documentation xml:lang="en"></xs:documentation>
+        <xs:documentation xml:lang="en"><![CDATA[
+The row (of the options) to choose the value from (by default -1, ie. last row)
+        ]]></xs:documentation>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="param_attribute" type="xs:string">
       <xs:annotation>
-        <xs:documentation xml:lang="en"></xs:documentation>
+        <xs:documentation xml:lang="en"><![CDATA[
+Applies to ``from_param``. The attribute of the parameter to use.
+        ]]></xs:documentation>
       </xs:annotation>
     </xs:attribute>
   </xs:complexType>
@@ -5806,7 +5814,7 @@ of this directive.
       ]]></xs:documentation>
     </xs:annotation>
     <xs:sequence>
-      <xs:element name="action" type="Action" minOccurs="1" maxOccurs="unbounded"/>
+      <xs:group ref="ActionsElement" minOccurs="1" maxOccurs="unbounded"/>
     </xs:sequence>
     <xs:attribute name="value" type="xs:string" use="optional">
       <xs:annotation>
@@ -5931,6 +5939,13 @@ Applies to ``param_value``, ``boolean``</xs:documentation>
         ]]></xs:documentation>
       </xs:annotation>
     </xs:attribute>
+    <xs:attribute name="iterate" type="PermissiveBoolean">
+      <xs:annotation>
+        <xs:documentation xml:lang="en"><![CDATA[
+Applies to ``insert_column``. Default is ``False``</xs:documentation>
+        ]]></xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
     <xs:attribute name="param_attribute" type="xs:string">
       <xs:annotation>
         <xs:documentation xml:lang="en">
@@ -5943,9 +5958,9 @@ Applies to ``param_value``</xs:documentation>
         <xs:documentation xml:lang="en">Applies to ``multiple_splitter``</xs:documentation>
       </xs:annotation>
     </xs:attribute>
-    <xs:attribute name="strip" type="PermissiveBoolean">
+    <xs:attribute name="strip" type="xs:string">
       <xs:annotation>
-        <xs:documentation xml:lang="en">Applies to ``column_strip``</xs:documentation>
+        <xs:documentation xml:lang="en">Applies to ``column_strip``. The given string is removed from the start or end of the column.</xs:documentation>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="old_column" type="xs:string">

lib/galaxy/webapps/galaxy/api/history_contents.py

@@ -382,7 +382,7 @@ class FastAPIHistoryContents:
     @router.get(
         "/api/histories/{history_id}/contents/{id}",
         name="history_content",
-        summary="Return detailed information about an HDA within a history.",
+        summary="Return detailed information about an HDA within a history. ``/api/histories/{history_id}/contents/{type}s/{id}`` should be used instead.",
         deprecated=True,
     )
     def show(
@@ -635,7 +635,7 @@ class FastAPIHistoryContents:
     )
     @router.put(
         "/api/histories/{history_id}/contents/{id}",
-        summary="Updates the values for the history content item with the given ``ID``.",
+        summary="Updates the values for the history content item with the given ``ID``. ``/api/histories/{history_id}/contents/{type}s/{id}`` should be used instead.",
         deprecated=True,
     )
     def update(

lib/galaxy_test/api/test_history_contents.py

@@ -1263,7 +1263,7 @@ class TestHistoryContentsApiBulkOperation(ApiTestCase):
         with self.dataset_populator.test_history() as history_id:
             self._create_test_history_contents(history_id)
 
-            invalid_filter_keys_with_stats = ["genome_build", "data_type", "annotation"]
+            invalid_filter_keys_with_stats = ["data_type", "annotation"]
 
             for filter_key in invalid_filter_keys_with_stats:
                 response = self._get_contents_with_stats(

lib/galaxy_test/api/test_tools.py

@@ -819,6 +819,23 @@ class TestToolsApi(ApiTestCase, TestsTools):
     def test_apply_rules_6(self):
         self._apply_rules_and_check(rules_test_data.EXAMPLE_6)
 
+    @skip_without_tool("galaxy_json_sleep")
+    def test_dataset_hidden_after_job_finish(self):
+        with self.dataset_populator.test_history() as history_id:
+            inputs = {
+                "sleep_time": 5,
+            }
+            response = self._run("galaxy_json_sleep", history_id, inputs, assert_ok=True)
+            output = response["outputs"][0]
+            response = self._put(
+                f"histories/{history_id}/contents/datasets/{output['id']}", data={"visible": False}, json=True
+            )
+            response.raise_for_status()
+            output_details = self.dataset_populator.get_history_dataset_details(history_id, dataset=output, wait=False)
+            assert not output_details["visible"]
+            output_details = self.dataset_populator.get_history_dataset_details(history_id, dataset=output, wait=True)
+            assert not output_details["visible"]
+
     @skip_without_tool("multi_select")
     def test_multi_select_as_list(self):
         with self.dataset_populator.test_history(require_new=False) as history_id:

lib/galaxy_test/selenium/test_collection_edit.py

@@ -14,6 +14,8 @@ class TestCollectionEdit(SeleniumTestCase):
         self.create_simple_list_collection()
         self.open_collection_edit_view()
         self.navigate_to_database_tab()
+        alert_element = self.components.edit_collection_attributes.alert_info.wait_for_visible()
+        assert "This will create a new collection in your History. Your quota will not increase." in alert_element.text
         dataValue = "unspecified"
         self.check_current_data_value(dataValue)
         dataNew = "hg17"
@@ -29,6 +31,11 @@ class TestCollectionEdit(SeleniumTestCase):
         self.create_simple_list_collection_txt()
         self.open_collection_edit_view()
         self.navigate_to_datatype_tab()
+        alert_element = self.components.edit_collection_attributes.alert_info.wait_for_visible()
+
+        assert (
+            "This operation might take a short while, depending on the size of your collection." in alert_element.text
+        )
         dataValue = "txt"
         self.check_current_data_value(dataValue)
         dataNew = "tabular"

static/welcome.html.sample

@@ -2,7 +2,7 @@
 <html lang="en">
     <head>
         <meta charset="utf-8">
-        <link rel="stylesheet" href="style/base.css" type="text/css" />
+        <link rel="stylesheet" href="dist/base.css" type="text/css" />
     </head>
     <body class="m-0">
         <div class="py-4">

templates/webapps/galaxy/visualization/phyloviz.mako

@@ -1,5 +1,7 @@
 <%inherit file="/base/base_panels.mako"/>
 ##
+<%def name="title()">${ config.get('title', 'Phyloviz')  + ' | Galaxy' }</%def>
+
 <%def name="init()">
     <%
         self.has_left_panel=False

test/functional/tools/galaxy_json_discover_and_sleep.xml

@@ -0,0 +1,28 @@
+<tool id="galaxy_json_sleep" name="galaxy_json_sleep" version="1.0.0" profile="22.05">
+    <command><![CDATA[ 
+        touch out &&
+        sleep $sleep_time &&
+        cp '$c1' galaxy.json
+    ]]></command>
+    <configfiles>
+        <configfile name="c1">{"output_tool_supplied_metadata": {
+            "name": "my dynamic name",
+            "ext": "txt",
+            "info": "my dynamic info"
+            }}
+</configfile>
+    </configfiles>
+    <inputs>
+        <param name="sleep_time" type="integer" value="0" />
+    </inputs>
+    <outputs>
+        <data name="output" format="txt" from_work_dir="out" />
+    </outputs>
+    <tests>
+        <test>
+            <param name="input" value="7" />
+            <output name="output" md5="d41d8cd98f00b204e9800998ecf8427e">
+            </output>
+        </test>
+    </tests>
+</tool>

test/functional/tools/sample_tool_conf.xml

@@ -1,15 +1,15 @@
 <?xml version="1.0"?>
 <toolbox tool_path="${tool_conf_dir}" is_shed_conf="false">
-  <tool file="upload.xml"/>
-  <tool file="export_remote.xml"/>
+  <tool file="upload.xml" />
+  <tool file="export_remote.xml" />
   <section id="test" name="Test Section">
     <tool file="multi_data_optional.xml" />
     <tool file="paths_as_file.xml" />
     <tool file="param_text_option.xml" />
     <tool file="column_param.xml" />
   </section>
-  <tool file="ucsc_tablebrowser.xml"/>
-  <tool file="test_data_source.xml"/>
+  <tool file="ucsc_tablebrowser.xml" />
+  <tool file="test_data_source.xml" />
   <tool file="simple_constructs.xml" />
   <tool file="color_param.xml" />
   <tool file="inheritance_simple.xml" />
@@ -19,10 +19,10 @@
   <tool file="environment_variables_inject.xml" />
   <tool file="code_file.xml" />
   <tool file="disambiguate_cond.xml" />
-  <tool file="multi_repeats.xml"/>
-  <tool file="library_data.xml"/>
-  <tool file="remove_value.xml"/>
-  <tool file="bibtex.xml"/>
+  <tool file="multi_repeats.xml" />
+  <tool file="library_data.xml" />
+  <tool file="remove_value.xml" />
+  <tool file="bibtex.xml" />
   <tool file="multi_select.xml" />
   <tool file="multi_output.xml" />
   <tool file="multi_output_configured.xml" />
@@ -30,6 +30,7 @@
   <tool file="multi_output_assign_primary_ext_dbkey.xml" />
   <tool file="multi_output_recurse.xml" />
   <tool file="multi_output_recurse_collection.xml" />
+  <tool file="galaxy_json_discover_and_sleep.xml" />
   <tool file="tool_provided_metadata_1.xml" />
   <tool file="tool_provided_metadata_2.xml" />
   <tool file="tool_provided_metadata_3.xml" />