Merge pull request #15744 from mvdbeek/dce_display

[23.0] Display DCE in job parameter component, allow rerunning with DCE input

client/src/components/History/Content/GenericItem.vue

@@ -3,19 +3,19 @@
         <loading-span v-if="loading" message="Loading dataset" />
         <div v-else>
             <ContentItem
-                :id="item.hid"
+                :id="item.hid ?? item.element_index + 1"
                 is-history-item
-                :item="item"
-                :name="item.name"
+                :item="item?.object || item"
+                :name="item.name || item.element_identifier"
                 :expand-dataset="expandDataset"
-                :is-dataset="item.history_content_type == 'dataset'"
+                :is-dataset="item.history_content_type == 'dataset' || item.element_type == 'hda'"
                 @update:expand-dataset="expandDataset = $event"
                 @view-collection="viewCollection = !viewCollection"
                 @delete="onDelete(item)"
                 @undelete="onUndelete(item)"
                 @unhide="onUnhide(item)" />
             <div v-if="viewCollection">
-                <GenericElement :dsc="item" />
+                <GenericElement :dsc="item?.object || item" />
             </div>
         </div>
     </component>
@@ -23,13 +23,15 @@
 
 <script>
 import LoadingSpan from "components/LoadingSpan";
-import { DatasetCollectionProvider, DatasetProvider } from "components/providers";
-import { deleteContent, updateContentFields } from "components/History/model/queries";
+import { DatasetCollectionProvider, DatasetProvider } from "@/components/providers";
+import { DatasetCollectionElementProvider } from "@/components/providers/storeProviders";
+import { deleteContent, updateContentFields } from "@/components/History/model/queries";
 import ContentItem from "./ContentItem";
 import GenericElement from "./GenericElement";
 
 export default {
     components: {
+        DatasetCollectionElementProvider,
         ContentItem,
         GenericElement,
         DatasetProvider,
@@ -54,7 +56,16 @@ export default {
     },
     computed: {
         providerComponent() {
-            return this.itemSrc == "hda" ? "DatasetProvider" : "DatasetCollectionProvider";
+            switch (this.itemSrc) {
+                case "hda":
+                    return "DatasetProvider";
+                case "hdca":
+                    return "DatasetCollectionProvider";
+                case "dce":
+                    return "DatasetCollectionElementProvider";
+                default:
+                    throw `Unknown element src ${this.itemSrc}`;
+            }
         },
     },
     methods: {

client/src/components/JobParameters/JobParametersArrayValue.vue

@@ -2,7 +2,7 @@
     <div>
         <div v-for="(elVal, pvIndex) in parameter_value" :key="pvIndex">
             <generic-history-item
-                v-if="['hda', 'hdca'].includes(elVal.src)"
+                v-if="['hda', 'hdca', 'dce'].includes(elVal.src)"
                 :item-id="elVal.id"
                 :item-src="elVal.src" />
             <span v-else> {{ elVal.hid }}: {{ elVal.name }} </span>

client/src/components/Tool/ToolForm.vue

@@ -155,7 +155,6 @@ export default {
     data() {
         return {
             disabled: false,
-            initialized: false,
             showLoading: true,
             showForm: false,
             showEntryPoints: false,
@@ -163,7 +162,7 @@ export default {
             showError: false,
             showExecuting: false,
             formConfig: {},
-            formData: {},
+            formData: undefined,
             remapAllowed: false,
             errorTitle: null,
             errorContent: null,
@@ -214,6 +213,9 @@ export default {
                 return "The previous run of this tool failed and other tools were waiting for it to finish successfully. Use this option to resume those tools using the new output(s) of this tool run.";
             }
         },
+        initialized() {
+            return this.formData !== undefined;
+        },
     },
     watch: {
         currentHistoryId() {
@@ -224,10 +226,7 @@ export default {
         },
     },
     created() {
-        this.requestTool().then(() => {
-            this.initialized = true;
-            console.debug(`ToolForm::created - Started listening to history changes. [${this.id}]`);
-        });
+        this.requestTool();
     },
     methods: {
         ...mapActions(useJobStore, ["saveLatestResponse"]),

client/src/components/providers/storeProviders.js

@@ -47,6 +47,7 @@ export const SimpleProviderMixin = {
             loading: this.loading,
             item: this.item,
             save: this.save,
+            result: this.item,
         });
     },
 };
@@ -131,6 +132,15 @@ export const JobProvider = {
     },
 };
 
+export const DatasetCollectionElementProvider = {
+    mixins: [SimpleProviderMixin],
+    computed: {
+        url() {
+            return prependPath(`api/dataset_collection_element/${this.id}`);
+        },
+    },
+};
+
 /**
  * Provider component interface to the actual stores i.e. history items and collection elements stores.
  * @param {String} storeAction The store action is executed when the consuming component e.g. the history panel, changes the provider props.

client/src/mvc/ui/ui-select-content.js

@@ -424,12 +424,12 @@ const View = Backbone.View.extend({
         const select_options = { hda: [], hdca: [] };
         _.each(options, (items, src) => {
             _.each(items, (item) => {
-                self._patchValue(item);
+                self._patchValue(item, src);
                 const current_src = item.src || src;
                 const addOption = !this.model.attributes.tag || item.tags.includes(this.model.attributes.tag);
                 if (addOption) {
                     select_options[current_src].push({
-                        hid: item.hid,
+                        hid: item.hid || Infinity, // if we got no hid we have a "Selected" item
                         keep: item.keep,
                         label: `${item.hid || "Selected"}: ${item.name}`,
                         value: item.id,
@@ -449,13 +449,19 @@ const View = Backbone.View.extend({
     _changeValue: function () {
         const new_value = this.model.get("value");
         if (new_value && new_value.values && new_value.values.length > 0) {
+            // sniff first suitable field type from config list
+            let src = new_value.values[0].src;
+            if (src === "dce") {
+                src =
+                    this.cache[`dce${new_value.values[0].id}_hda`]?.src ||
+                    this.cache[`dce${new_value.values[0].id}_hdca`]?.src;
+            }
+            this._patchValue(new_value, src);
             // create list with content ids
             const list = [];
             _.each(new_value.values, (value) => {
                 list.push(value.id);
             });
-            // sniff first suitable field type from config list
-            const src = new_value.values[0].src;
             const multiple = new_value.values.length > 1;
             for (let i = 0; i < this.config.length; i++) {
                 const field = this.fields[i];
@@ -480,11 +486,11 @@ const View = Backbone.View.extend({
 
     /** Library datasets are displayed and selected together with history datasets,
         Dataset collection elements are displayed together with history dataset collections **/
-    _patchValue: function (v) {
-        const patchTo = { ldda: "hda", dce: "hdca" };
+    _patchValue: function (v, src) {
+        const patchTo = { ldda: "hda", dce: src };
         if (v.values) {
             _.each(v.values, (v) => {
-                this._patchValue(v);
+                this._patchValue(v, src);
             });
         } else if (patchTo[v.src]) {
             v.origin = v.src;

client/src/schema/schema.ts

@@ -47,6 +47,10 @@ export interface paths {
          */
         put: operations["reload_toolbox_api_configuration_toolbox_put"];
     };
+    "/api/dataset_collection_element/{dce_id}": {
+        /** Content */
+        get: operations["content_api_dataset_collection_element__dce_id__get"];
+    };
     "/api/dataset_collections": {
         /** Create a new dataset collection instance. */
         post: operations["create_api_dataset_collections_post"];
@@ -7544,6 +7548,33 @@ export interface operations {
             };
         };
     };
+    content_api_dataset_collection_element__dce_id__get: {
+        /** Content */
+        parameters: {
+            /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
+            header?: {
+                "run-as"?: string;
+            };
+            /** @description The encoded identifier of the dataset collection element. */
+            path: {
+                dce_id: string;
+            };
+        };
+        responses: {
+            /** @description Successful Response */
+            200: {
+                content: {
+                    "application/json": components["schemas"]["DCESummary"];
+                };
+            };
+            /** @description Validation Error */
+            422: {
+                content: {
+                    "application/json": components["schemas"]["HTTPValidationError"];
+                };
+            };
+        };
+    };
     create_api_dataset_collections_post: {
         /** Create a new dataset collection instance. */
         parameters?: {

client/src/utils/navigation/navigation.yml

@@ -242,6 +242,7 @@ history_panel:
       tag_area_button: '.details .stateless-tags .multiselect button'
       tag_area_input: '.details .stateless-tags .multiselect input'
       list_items: '.dataset-collection-panel .listing .content-item'
+      back_to_history: svg[data-description="back to history"]
 
   selectors:
     _: '#current-history-panel'
@@ -492,6 +493,11 @@ tool_form:
     parameter_checkbox: 'div.ui-form-element[id="form-element-${parameter}"] .ui-switch'
     parameter_input: 'div.ui-form-element[id="form-element-${parameter}"] .ui-input'
     parameter_textarea: 'div.ui-form-element[id="form-element-${parameter}"] textarea'
+    parameter_batch_dataset_collection:
+      type: xpath
+      selector: //div[@id='form-element-${parameter}']//i[contains(@class, 'fa-folder-o')]/parent::label
+    data_option_value: option[value="${item_id}"]
+
     repeat_insert: '[data-description="repeat insert"]'
     reference: '.formatted-reference'
     about: '.tool-footer'

lib/galaxy/managers/jobs.py

@@ -903,7 +903,7 @@ def summarize_job_parameters(trans, job):
                             value=f"{len(param_values[input.name])} uploaded datasets",
                         )
                     )
-                elif input.type == "data":
+                elif input.type == "data" or input.type == "data_collection":
                     value = []
                     for element in listify(param_values[input.name]):
                         encoded_id = trans.security.encode_id(element.id)

lib/galaxy/model/security.py

@@ -4,6 +4,7 @@ from datetime import (
     datetime,
     timedelta,
 )
+from typing import List
 
 from sqlalchemy import (
     and_,
@@ -519,6 +520,12 @@ class GalaxyRBACAgent(RBACAgent):
 
         return True
 
+    def can_access_collection(self, user_roles: List[galaxy.model.Role], collection: galaxy.model.DatasetCollection):
+        action_tuples = collection.dataset_action_tuples
+        if not self.can_access_datasets(user_roles, action_tuples):
+            return False
+        return True
+
     def can_manage_dataset(self, roles, dataset):
         return self.allow_action(roles, self.permitted_actions.DATASET_MANAGE_PERMISSIONS, dataset)
 

lib/galaxy/tools/__init__.py

@@ -2439,12 +2439,6 @@ class Tool(Dictifiable):
         # create parameter object
         params = Params(kwd, sanitize=False)
 
-        # expand incoming parameters (parameters might trigger multiple tool executions,
-        # here we select the first execution only in order to resolve dynamic parameters)
-        expanded_incomings, _ = expand_meta_parameters(trans, self, params.__dict__)
-        if expanded_incomings:
-            params.__dict__ = expanded_incomings[0]
-
         # do param translation here, used by datasource tools
         if self.input_translator:
             self.input_translator.translate(params)

lib/galaxy/tools/parameters/basic.py

@@ -2092,6 +2092,9 @@ class DataToolParameter(BaseDataToolParameter):
                     elif single_value["src"] == "ldda":
                         decoded_id = trans.security.decode_id(single_value["id"])
                         rval.append(trans.sa_session.query(LibraryDatasetDatasetAssociation).get(decoded_id))
+                    elif single_value["src"] == "dce":
+                        decoded_id = trans.security.decode_id(single_value["id"])
+                        rval.append(trans.sa_session.query(DatasetCollectionElement).get(decoded_id))
                     else:
                         raise ValueError(f"Unknown input source {single_value['src']} passed to job submission API.")
                 elif isinstance(
@@ -2127,6 +2130,9 @@ class DataToolParameter(BaseDataToolParameter):
             elif value["src"] == "hdca":
                 decoded_id = trans.security.decode_id(value["id"])
                 rval.append(trans.sa_session.query(HistoryDatasetCollectionAssociation).get(decoded_id))
+            elif value["src"] == "dce":
+                decoded_id = trans.security.decode_id(value["id"])
+                rval.append(trans.sa_session.query(DatasetCollectionElement).get(decoded_id))
             else:
                 raise ValueError(f"Unknown input source {value['src']} passed to job submission API.")
         elif str(value).startswith("__collection_reduce__|"):
@@ -2280,6 +2286,28 @@ class DataToolParameter(BaseDataToolParameter):
                 value["map_over_type"] = subcollection_type
             return list.append(value)
 
+        def append_dce(dce):
+            if dce.hda:
+                # well this isn't good, but what's the alternative ?
+                # we should be precise about what we're (re-)running here.
+                key = "hda"
+            else:
+                key = "hdca"
+            d["options"][key].append(
+                {
+                    "id": trans.security.encode_id(dce.id),
+                    "name": dce.element_identifier,
+                    "src": "dce",
+                    "tags": [],
+                    "keep": True,
+                }
+            )
+
+        # append DCE
+        if isinstance(other_values.get(self.name), DatasetCollectionElement):
+            dce = other_values[self.name]
+            append_dce(dce)
+
         # add datasets
         hda_list = util.listify(other_values.get(self.name))
         # Prefetch all at once, big list of visible, non-deleted datasets.
@@ -2299,6 +2327,8 @@ class DataToolParameter(BaseDataToolParameter):
                 else:
                     hda_state = "unavailable"
                 append(d["options"]["hda"], hda, f"({hda_state}) {hda.name}", "hda", True)
+            elif isinstance(hda, DatasetCollectionElement):
+                append_dce(hda)
 
         # add dataset collections
         dataset_collection_matcher = dataset_matcher_factory.dataset_collection_matcher(dataset_matcher)
@@ -2460,7 +2490,7 @@ class DataCollectionToolParameter(BaseDataToolParameter):
         # append DCE
         if isinstance(other_values.get(self.name), DatasetCollectionElement):
             dce = other_values[self.name]
-            d["options"]["dce"].append(
+            d["options"]["hdca"].append(
                 {
                     "id": trans.security.encode_id(dce.id),
                     "hid": None,

lib/galaxy/webapps/galaxy/api/dataset_collections.py

@@ -13,6 +13,7 @@ from galaxy.schema.schema import (
     AnyHDCA,
     CreateNewCollectionPayload,
     DatasetCollectionInstanceType,
+    DCESummary,
     HDCADetailed,
 )
 from galaxy.webapps.galaxy.api import (
@@ -36,6 +37,11 @@ DatasetCollectionIdPathParam: DecodedDatabaseIdField = Path(
     ..., description="The encoded identifier of the dataset collection."
 )
 
+
+DatasetCollectionElementIdPathParam: DecodedDatabaseIdField = Path(
+    ..., description="The encoded identifier of the dataset collection element."
+)
+
 InstanceTypeQueryParam: DatasetCollectionInstanceType = Query(
     default="history",
     description="The type of collection instance. Either `history` (default) or `library`.",
@@ -129,3 +135,11 @@ class FastAPIDatasetCollections:
         ),
     ) -> DatasetCollectionContentElements:
         return self.service.contents(trans, hdca_id, parent_id, instance_type, limit, offset)
+
+    @router.get("/api/dataset_collection_element/{dce_id}")
+    def content(
+        self,
+        trans: ProvidesHistoryContext = DependsOnTrans,
+        dce_id: DecodedDatabaseIdField = DatasetCollectionElementIdPathParam,
+    ) -> DCESummary:
+        return self.service.dce_content(trans, dce_id)

lib/galaxy/webapps/galaxy/services/dataset_collections.py

@@ -25,6 +25,7 @@ from galaxy.managers.collections_util import (
 from galaxy.managers.context import ProvidesHistoryContext
 from galaxy.managers.hdcas import HDCAManager
 from galaxy.managers.histories import HistoryManager
+from galaxy.model import DatasetCollectionElement
 from galaxy.schema.fields import (
     DecodedDatabaseIdField,
     ModelClassField,
@@ -208,6 +209,17 @@ class DatasetCollectionsService(ServiceBase, UsesLibraryMixinItems):
         )
         return rval
 
+    def dce_content(self, trans: ProvidesHistoryContext, dce_id: DecodedDatabaseIdField) -> DCESummary:
+        dce: Optional[DatasetCollectionElement] = trans.model.session.query(DatasetCollectionElement).get(dce_id)
+        if not dce:
+            raise exceptions.ObjectNotFound("No DatasetCollectionElement found")
+        if not trans.user_is_admin:
+            collection = dce.child_collection or dce.collection
+            if not trans.app.security_agent.can_access_collection(trans.get_current_user_roles(), collection):
+                raise exceptions.ItemAccessibilityException("Collection not accessible by user.")
+        serialized_dce = dictify_element_reference(dce, recursive=False, security=trans.security)
+        return trans.security.encode_all_ids(serialized_dce, recursive=True)
+
     def contents(
         self,
         trans: ProvidesHistoryContext,

lib/galaxy_test/api/test_dataset_collections.py

@@ -196,6 +196,40 @@ class TestDatasetCollectionsApi(ApiTestCase):
                 create_response = self._post("dataset_collections", payload, json=True)
                 self._assert_status_code_is(create_response, 403)
 
+    def test_dataset_collection_element_security(self):
+        with self.dataset_populator.test_history(require_new=False) as history_id:
+            dataset_collection = self.dataset_collection_populator.create_list_of_list_in_history(
+                history_id,
+                collection_type="list:list:list",
+                wait=True,
+            ).json()
+            first_element = dataset_collection["elements"][0]
+            assert first_element["model_class"] == "DatasetCollectionElement"
+            assert first_element["element_type"] == "dataset_collection"
+            first_element_url = f"/api/dataset_collection_element/{first_element['id']}"
+            # Make one dataset private to check that access permissions are respected
+            first_dataset_element = first_element["object"]["elements"][0]["object"]["elements"][0]
+            self.dataset_populator.make_private(history_id, first_dataset_element["object"]["id"])
+            with self._different_user():
+                assert self._get(first_element_url).status_code == 403
+            collection_dce_response = self._get(first_element_url)
+            collection_dce_response.raise_for_status()
+            collection_dce = collection_dce_response.json()
+            assert collection_dce["model_class"] == "DatasetCollectionElement"
+            assert collection_dce["element_type"] == "dataset_collection"
+            first_dataset_element = first_element["object"]["elements"][0]["object"]["elements"][0]
+            assert first_dataset_element["model_class"] == "DatasetCollectionElement"
+            assert first_dataset_element["element_type"] == "hda"
+            first_dataset_element_url = f"/api/dataset_collection_element/{first_dataset_element['id']}"
+            with self._different_user():
+                assert self._get(first_dataset_element_url).status_code == 403
+            dataset_dce_response = self._get(first_dataset_element_url)
+            dataset_dce_response.raise_for_status()
+            dataset_dce = dataset_dce_response.json()
+            assert dataset_dce["model_class"] == "DatasetCollectionElement"
+            assert dataset_dce["element_type"] == "hda"
+            assert dataset_dce["object"]["model_class"] == "HistoryDatasetAssociation"
+
     def test_enforces_unique_names(self):
         with self.dataset_populator.test_history(require_new=False) as history_id:
             element_identifiers = self.dataset_collection_populator.list_identifiers(history_id)

lib/galaxy_test/selenium/test_tool_form.py

@@ -114,6 +114,38 @@ class TestToolForm(SeleniumTestCase, UsesHistoryItemAssertions):
         self.history_panel_wait_for_hid_ok(2)
         self._check_dataset_details_for_inttest_value(2)
 
+    @selenium_test
+    def test_rerun_dataset_collection_element(self):
+        # upload a first dataset that should not become selected on re-run
+        test_path = self.get_filename("1.fasta")
+        self.perform_upload(test_path)
+        self.history_panel_wait_for_hid_ok(1)
+
+        history_id = self.current_history_id()
+        # upload a nested collection
+        collection_id = self.dataset_collection_populator.create_list_of_list_in_history(
+            history_id,
+            collection_type="list:list",
+            wait=True,
+        ).json()["id"]
+        self.tool_open("identifier_multiple")
+        self.components.tool_form.parameter_batch_dataset_collection(parameter="input1").wait_for_and_click()
+        self.sleep_for(self.wait_types.UX_RENDER)
+        self.components.tool_form.data_option_value(item_id=collection_id).wait_for_and_click()
+        self.sleep_for(self.wait_types.UX_RENDER)
+        self.tool_form_execute()
+        self.history_panel_wait_for_hid_ok(7)
+        self.history_panel_expand_collection(7)
+        self.sleep_for(self.wait_types.UX_RENDER)
+        self.history_panel_click_item_title(1)
+        self.sleep_for(self.wait_types.UX_RENDER)
+        self.hda_click_primary_action_button(1, "rerun")
+        self.sleep_for(self.wait_types.UX_RENDER)
+        assert self.driver.find_element(By.CSS_SELECTOR, "option:checked").text == "Selected: test0"
+        self.tool_form_execute()
+        self.components.history_panel.collection_view.back_to_history.wait_for_and_click()
+        self.history_panel_wait_for_hid_ok(9)
+
     @selenium_test
     @flakey
     def test_run_data(self):