Merge branch 'release_22.05' into dev

client/src/components/FilesDialog/FilesDialog.vue

@@ -4,7 +4,7 @@
         :options-show="optionsShow"
         :modal-show="modalShow"
         :hide-modal="() => (modalShow = false)"
-        :back-func="load"
+        :back-func="goBack"
         :undo-show="undoShow">
         <template v-slot:search>
             <data-dialog-search v-model="filter" />
@@ -330,6 +330,10 @@ export default {
                     });
             }
         },
+        goBack() {
+            // Loading without a record navigates back one level
+            this.load();
+        },
         parseItemFileMode(item) {
             const result = {
                 id: item.uri,

client/src/components/History/Content/model/states.js

@@ -11,7 +11,7 @@ export const STATES = {
     },
     deferred: {
         status: "info",
-        text: "This dataset is remote, has not be ingested by Galaxy, and full metadata may not be available.",
+        text: "This dataset is remote, has not been ingested by Galaxy, and full metadata may not be available.",
         icon: "cloud",
     },
     /** has no data */

client/src/components/History/CurrentHistory/HistoryNavigation.vue

@@ -130,7 +130,7 @@
         <SelectorModal
             id="selector-history-modal"
             :histories="histories"
-            :current-history="history"
+            :current-history-id="history.id"
             @selectHistory="$emit('setCurrentHistory', $event)" />
 
         <CopyModal id="copy-history-modal" :history="history" />

client/src/components/History/Modals/SelectorModal.test.js

@@ -0,0 +1,65 @@
+import { mount } from "@vue/test-utils";
+import flushPromises from "flush-promises";
+import { getLocalVue } from "jest/helpers";
+import SelectorModal from "./SelectorModal";
+
+const localVue = getLocalVue();
+
+const SELECTED_HISTORY_ID = "COOL_ID";
+const getFakeHistorySummaries = (num, selectedIndex = 0) => {
+    const result = Array.from({ length: num }, (_, index) => ({
+        id: `ID-${index}`,
+        name: `History-${index}`,
+        tags: [],
+        update_time: new Date().toISOString(),
+    }));
+    result[selectedIndex].id = SELECTED_HISTORY_ID;
+    return result;
+};
+const PROPS_WITH_10_HISTORIES = {
+    currentHistoryId: SELECTED_HISTORY_ID,
+    histories: getFakeHistorySummaries(10),
+    perPage: 3,
+    static: true, // Force the modal visible for testing
+};
+
+describe("History SelectorModal.vue", () => {
+    let wrapper;
+
+    async function mountWith(props) {
+        wrapper = mount(SelectorModal, {
+            propsData: props,
+            localVue,
+        });
+        await flushPromises();
+    }
+
+    it("should highlight the currently selected history", async () => {
+        await mountWith(PROPS_WITH_10_HISTORIES);
+
+        const selectedRows = wrapper.findAll(".table-success");
+        expect(selectedRows.length).toBe(1);
+        expect(selectedRows.at(0).attributes("data-pk")).toBe(SELECTED_HISTORY_ID);
+    });
+
+    it("paginates the histories", async () => {
+        await mountWith(PROPS_WITH_10_HISTORIES);
+
+        const displayedRows = wrapper.findAll("tbody > tr").wrappers;
+        expect(displayedRows.length).toBe(3);
+        expect(wrapper.vm.histories.length).toBe(10);
+    });
+
+    it("emits selectHistory with the correct history ID when a row is clicked", async () => {
+        await mountWith(PROPS_WITH_10_HISTORIES);
+
+        expect(wrapper.emitted()["selectHistory"]).toBeUndefined();
+
+        const targetHistoryId = "ID-2";
+        const targetRow = wrapper.find(`[data-pk="${targetHistoryId}"]`);
+        await targetRow.trigger("click");
+
+        expect(wrapper.emitted()["selectHistory"]).toBeDefined();
+        expect(wrapper.emitted()["selectHistory"][0][0].id).toBe(targetHistoryId);
+    });
+});

client/src/components/History/Modals/SelectorModal.vue

@@ -1,25 +1,24 @@
 <template>
     <b-modal ref="modal" v-bind="$attrs" :title="'Switch to History' | l" v-on="$listeners">
         <b-form-group :description="'Filter histories' | l">
-            <b-input v-model="filter" :placeholder="'Search Filter' | l" />
+            <b-form-input v-model="filter" type="search" :placeholder="'Search Filter' | l" />
         </b-form-group>
 
         <b-table
-            ref="history-list"
-            v-model="currentRows"
             striped
             hover
             sticky-header="50vh"
             primary-key="id"
             :fields="fields"
             :filter="filter"
-            :items="histories"
+            :items="formattedItems"
             :per-page="perPage"
             :current-page="currentPage"
             :selectable="true"
             select-mode="single"
             selected-variant="success"
-            @row-selected="switchToHistory">
+            @row-selected="switchToHistory"
+            @filtered="onFiltered">
             <template v-slot:cell(tags)="row">
                 <stateless-tags :value="row.item.tags" :disabled="true" />
             </template>
@@ -29,23 +28,28 @@
         </b-table>
 
         <template v-slot:modal-footer>
-            <b-pagination v-model="currentPage" :total-rows="filteredRowCount" :per-page="perPage"></b-pagination>
+            <b-pagination v-model="currentPage" :total-rows="totalRows" :per-page="perPage" />
         </template>
     </b-modal>
 </template>
 
 <script>
+import { BModal, BFormGroup, BFormInput, BTable, BPagination } from "bootstrap-vue";
 import { StatelessTags } from "components/Tags";
 import UtcDate from "components/UtcDate";
-import { debounce } from "underscore";
 
 export default {
     components: {
         StatelessTags,
         UtcDate,
+        BModal,
+        BFormGroup,
+        BFormInput,
+        BTable,
+        BPagination,
     },
     props: {
-        currentHistory: { type: Object, required: true },
+        currentHistoryId: { type: String, required: true },
         histories: { type: Array, default: () => [] },
         perPage: { type: Number, required: false, default: 50 },
     },
@@ -53,30 +57,22 @@ export default {
         return {
             filter: null,
             currentPage: 1,
-            currentRows: [],
+            totalRows: 0,
         };
     },
     computed: {
-        filteredRowCount() {
-            return this.currentRows.length;
-        },
-        selectedIndex() {
-            return this.currentRows.findIndex((h) => h.id == this.currentHistory.id);
+        formattedItems() {
+            return this.histories.map((item) => {
+                if (item.id == this.currentHistoryId) {
+                    item._rowVariant = "success";
+                }
+                return item;
+            });
         },
     },
     watch: {
-        currentRows() {
-            this.selectCurrentRow();
-        },
-        filteredRowCount(newVal, oldVal) {
-            if (newVal != oldVal) {
-                this.currentPage = 1;
-            }
-        },
-        selectedIndex(idx, oldIdx) {
-            if (idx != oldIdx) {
-                this.debounceSelectCurrentRow();
-            }
+        histories(newVal) {
+            this.totalRows = newVal.length;
         },
     },
     created() {
@@ -85,7 +81,6 @@ export default {
             { key: "tags", sortable: true },
             { key: "update_time", label: "Updated", sortable: true },
         ];
-        this.debounceSelectCurrentRow = debounce(this.selectCurrentRow, 100);
     },
     methods: {
         switchToHistory(selected) {
@@ -93,12 +88,9 @@ export default {
                 this.$emit("selectHistory", selected[0]);
             }
         },
-        selectCurrentRow() {
-            const idx = this.selectedIndex;
-            const list = this.$refs["history-list"];
-            if (list && idx > -1) {
-                list.selectRow(idx);
-            }
+        onFiltered(filteredItems) {
+            this.totalRows = filteredItems.length;
+            this.currentPage = 1;
         },
     },
 };

client/src/components/Tool/ToolCard.vue

@@ -65,7 +65,7 @@
                         <span class="fa fa-star-o" />
                     </b-button>
                     <b-button
-                        v-else
+                        v-else-if="showRemoveFavorite"
                         v-b-tooltip.hover
                         role="button"
                         title="Remove from Favorites"
@@ -176,8 +176,14 @@ export default {
         versions() {
             return this.options.versions;
         },
+        hasUser() {
+            return !this.user.isAnonymous;
+        },
         showAddFavorite() {
-            return !!this.user.email && !this.isFavorite;
+            return this.hasUser && !this.isFavorite;
+        },
+        showRemoveFavorite() {
+            return this.hasUser && !this.showAddFavorite;
         },
         showVersions() {
             return this.versions && this.versions.length > 1;

doc/source/releases/22.01_announce_user.rst

@@ -14,9 +14,7 @@ Do you use tools which require colour inputs, like Circos? Previously we had a r
 
 .. raw:: html
 
-   <video controls autoplay loop muted width="100%" alt="Gif of the colour selector in action, a mouse clicks on the colour selector and gone is the restricted 20 colour palette, now there is a colour field and you can select any possible RGB colour.">
-       <source src="../_static/22.01-colour.mp4" type="video/mp4">
-   </video>
+   <iframe width="560" height="315" src="https://www.youtube-nocookie.com/embed/DSnFa5012RE" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen alt="Gif of the colour selector in action, a mouse clicks on the colour selector and gone is the restricted 20 colour palette, now there is a colour field and you can select any possible RGB colour."></iframe>
 
 
 **Improved File Export**
@@ -25,9 +23,7 @@ If you've been exporting files from Galaxy lately, you've probably seen the amaz
 
 .. raw:: html
 
-   <video controls autoplay loop muted width="100%" alt="Video of a user exporting a file using the new file export, selecting a folder, creating some subdirectories, before changing their mind and choosing another location to export their file to.">
-       <source src="../_static/22.01-export.mp4" type="video/mp4">
-   </video>
+   <iframe width="560" height="315" src="https://www.youtube-nocookie.com/embed/KBFrgWGlPus" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen alt="Video of a user exporting a file using the new file export, selecting a folder, creating some subdirectories, before changing their mind and choosing another location to export their file to."></iframe>
 
 **Improved File Uploads**
 
@@ -142,7 +138,7 @@ A special thanks to the release testing team for testing many of the new feature
 - `Natalie Kucher <https://github.com/nakucher>`_
 - `Alex Ostrovsky <https://github.com/astrovsky01>`_
 - `Enis Afgan <https://github.com/afgane>`_
- 
+
 Release Notes
 ===========================================================
 

lib/galaxy/managers/histories.py

@@ -482,9 +482,11 @@ class HistorySerializer(sharable.SharableModelSerializer, deletable.PurgableSeri
             "size": lambda item, key, **context: int(item.disk_size),
             "nice_size": lambda item, key, **context: item.disk_nice_size,
             "state": self.serialize_history_state,
-            "url": lambda item, key, **context: self.url_for("history", id=self.app.security.encode_id(item.id)),
+            "url": lambda item, key, **context: self.url_for(
+                "history", id=self.app.security.encode_id(item.id), context=context
+            ),
             "contents_url": lambda item, key, **context: self.url_for(
-                "history_contents", history_id=self.app.security.encode_id(item.id)
+                "history_contents", history_id=self.app.security.encode_id(item.id), context=context
             ),
             "empty": lambda item, key, **context: (len(item.datasets) + len(item.dataset_collections)) <= 0,
             "count": lambda item, key, **context: len(item.datasets),

lib/galaxy/metadata/set_metadata.py

@@ -434,7 +434,7 @@ def set_metadata_portable(
                 if not object_store or not export_store:
                     # Can't happen, but type system doesn't know
                     raise Exception("object_store not built")
-                if not link_data_only and os.path.getsize(external_filename):
+                if not is_deferred and not link_data_only and os.path.getsize(external_filename):
                     # Here we might be updating a disk based objectstore when outputs_to_working_directory is used,
                     # or a remote object store from its cache path.
                     object_store.update_from_file(dataset.dataset, file_name=external_filename, create=True)
@@ -469,7 +469,7 @@ def set_metadata_portable(
                         context_value = context[context_key]
                         setattr(dataset, context_key, context_value)
                 # We only want to persist the external_filename if the dataset has been linked in.
-                if not link_data_only:
+                if not is_deferred and not link_data_only:
                     dataset.dataset.external_filename = None
                     dataset.dataset.extra_files_path = None
                 export_store.add_dataset(dataset)

lib/galaxy/model/deferred.py

@@ -40,6 +40,7 @@ log = logging.getLogger(__name__)
 class TransientDatasetPaths(NamedTuple):
     external_filename: str
     external_extra_files_path: str
+    metadata_files_dir: str
 
 
 class TransientPathMapper:
@@ -61,7 +62,7 @@ class SimpleTransientPathMapper(TransientPathMapper):
         external_filename = os.path.join(self._staging_directory, external_filename_basename)
         external_extras_basename = "dataset_%s_files" % str(old_dataset.uuid)
         external_extras = os.path.join(self._staging_directory, external_extras_basename)
-        return TransientDatasetPaths(external_filename, external_extras)
+        return TransientDatasetPaths(external_filename, external_extras, self._staging_directory)
 
 
 class DatasetInstanceMaterializer:
@@ -110,6 +111,7 @@ class DatasetInstanceMaterializer:
         materialized_dataset.hashes = [h.copy() for h in dataset.hashes]
 
         target_source = self._find_closest_dataset_source(dataset)
+        transient_paths = None
         if attached:
             object_store_populator = self._object_store_populator
             assert object_store_populator
@@ -157,12 +159,14 @@ class DatasetInstanceMaterializer:
             materialized_dataset_instance.has_metadata_files or materialized_dataset_instance.metadata_deferred
         )
         if require_metadata_regeneration:
-            if attached and self._sa_session:
-                # as of mid April 2022, we now get JSON encoding errors if this
-                # isn't bound to the session before metadata generation.
-                self._sa_session.add(materialized_dataset_instance)
             materialized_dataset_instance.init_meta()
-            materialized_dataset_instance.set_meta()
+            if transient_paths:
+                metadata_tmp_files_dir = transient_paths.metadata_files_dir
+            else:
+                # If metadata_tmp_files_dir is set we generate a MetdataTempFile,
+                # which we don't want when we're generating an attached materialized dataset instance
+                metadata_tmp_files_dir = None
+            materialized_dataset_instance.set_meta(metadata_tmp_files_dir=metadata_tmp_files_dir)
             materialized_dataset_instance.metadata_deferred = False
         return materialized_dataset_instance
 

lib/galaxy/webapps/galaxy/api/histories.py

@@ -180,6 +180,7 @@ class FastAPIHistories:
 
     @router.get(
         "/api/histories/{id}",
+        name="history",
         summary="Returns the history with the given ID.",
     )
     def show(

lib/galaxy/webapps/galaxy/api/history_contents.py

@@ -713,6 +713,12 @@ class FastAPIHistoryContents:
             description="When deleting a dataset collection, whether to also delete containing datasets.",
             deprecated=True,
         ),
+        stop_job: Optional[bool] = Query(
+            default=False,
+            title="Stop Job",
+            description="Whether to stop the creating job if all outputs of the job have been deleted.",
+            deprecated=True,
+        ),
         payload: DeleteHistoryContentPayload = Body(None),
     ):
         """
@@ -725,6 +731,7 @@ class FastAPIHistoryContents:
             payload = DeleteHistoryContentPayload()
         payload.purge = payload.purge or purge is True
         payload.recursive = payload.recursive or recursive is True
+        payload.stop_job = payload.stop_job or stop_job is True
         rval = self.service.delete(
             trans,
             id=id,

lib/galaxy_test/api/test_datasets.py

@@ -338,6 +338,13 @@ class DatasetsApiTestCase(ApiTestCase):
 
     @skip_without_tool("cat_data_and_sleep")
     def test_delete_cancels_job(self):
+        self._run_cancel_job(use_query_params=False)
+
+    @skip_without_tool("cat_data_and_sleep")
+    def test_delete_cancels_job_with_query_params(self):
+        self._run_cancel_job(use_query_params=True)
+
+    def _run_cancel_job(self, use_query_params=False):
         hda_id = self.dataset_populator.new_dataset(self.history_id)["id"]
         inputs = {
             "input1": {"src": "hda", "id": hda_id},
@@ -355,7 +362,9 @@ class DatasetsApiTestCase(ApiTestCase):
         assert job_details["state"] in ("new", "queued", "running"), job_details
 
         # Use stop_job to cancel the creating job
-        delete_response = self.dataset_populator.delete_dataset(self.history_id, output_hda_id, stop_job=True)
+        delete_response = self.dataset_populator.delete_dataset(
+            self.history_id, output_hda_id, stop_job=True, use_query_params=use_query_params
+        )
         self._assert_status_code_is_ok(delete_response)
         deleted_hda = delete_response.json()
         assert deleted_hda["deleted"], deleted_hda

lib/galaxy_test/api/test_histories.py

@@ -93,6 +93,15 @@ class HistoriesApiTestCase(ApiTestCase, BaseHistories):
         self._assert_has_keys(state_details, *states)
         self._assert_has_keys(state_ids, *states)
 
+    def test_show_history_returns_expected_urls(self):
+        # This test can be dropped when the URL attributes become deprecated
+        history_id = self._create_history("TestHistoryForUrls")["id"]
+        show_response = self._show(history_id)
+        self._assert_has_key(show_response, "id", "url", "contents_url")
+
+        assert show_response["url"] == f"/api/histories/{history_id}"
+        assert show_response["contents_url"] == f"/api/histories/{history_id}/contents"
+
     def test_show_most_recently_used(self):
         history_id = self._create_history("TestHistoryRecent")["id"]
         show_response = self._get("histories/most_recently_used").json()

lib/galaxy_test/api/test_tools.py

@@ -2729,6 +2729,54 @@ class ToolsTestCase(ApiTestCase, TestsTools):
         output_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output)
         assert output_content.startswith("chr1	147962192	147962580	CCDS989.1_cds_0_0_chr1_147962193_r	0	-")
 
+    @skip_without_tool("metadata_bam")
+    @uses_test_history(require_new=False)
+    def test_run_deferred_dataset_with_metadata_options_filter(self, history_id):
+        details = self.dataset_populator.create_deferred_hda(
+            history_id, "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bam", ext="bam"
+        )
+        inputs = {"input_bam": dataset_to_param(details), "ref_names": "chrM"}
+        run_response = self.dataset_populator.run_tool(tool_id="metadata_bam", inputs=inputs, history_id=history_id)
+        output = run_response["outputs"][0]
+        output_details = self.dataset_populator.get_history_dataset_details(
+            history_id, dataset=output, wait=True, assert_ok=True
+        )
+        assert output_details["state"] == "ok"
+        output_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output)
+        assert output_content.startswith("chrM")
+
+    @skip_without_tool("pileup")
+    @uses_test_history(require_new=False)
+    def test_metadata_validator_on_deferred_input(self, history_id):
+        deferred_bam_details = self.dataset_populator.create_deferred_hda(
+            history_id, "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bam", ext="bam"
+        )
+        fasta1_contents = open(self.get_filename("1.fasta")).read()
+        fasta = self.dataset_populator.new_dataset(history_id, content=fasta1_contents)
+        inputs = {"input1": dataset_to_param(deferred_bam_details), "reference": dataset_to_param(fasta)}
+        run_response = self.dataset_populator.run_tool(tool_id="pileup", inputs=inputs, history_id=history_id)
+        self.dataset_populator.wait_for_job(run_response["jobs"][0]["id"], assert_ok=True)
+
+    @pytest.mark.xfail
+    @skip_without_tool("pileup")
+    @uses_test_history(require_new=False)
+    def test_metadata_validator_can_fail_on_deferred_input(self, history_id):
+        # This test fails because we just skip the validator
+        # Fixing this is a TODO
+        deferred_bam_details = self.dataset_populator.create_deferred_hda(
+            history_id,
+            "https://github.com/galaxyproject/galaxy/blob/dev/test-data/3unsorted.bam?raw=true",
+            ext="unsorted.bam",
+        )
+        fasta1_contents = open(self.get_filename("1.fasta")).read()
+        fasta = self.dataset_populator.new_dataset(history_id, content=fasta1_contents)
+        inputs = {"input1": dataset_to_param(deferred_bam_details), "reference": dataset_to_param(fasta)}
+        run_response = self.dataset_populator.run_tool(tool_id="pileup", inputs=inputs, history_id=history_id)
+        self.dataset_populator.wait_for_job(run_response["jobs"][0]["id"], assert_ok=False)
+        job_id = run_response["jobs"][0]["id"]
+        job_details = self.dataset_populator.get_job_details(job_id=job_id).json()
+        assert job_details["state"] == "failed"
+
     @skip_without_tool("cat1")
     @uses_test_history(require_new=False)
     def test_run_deferred_mapping(self, history_id: str):

lib/galaxy_test/base/populators.py

@@ -597,9 +597,13 @@ class BaseDatasetPopulator(BasePopulator):
         purge: bool = False,
         stop_job: bool = False,
         wait_for_purge: bool = False,
+        use_query_params: bool = False,
     ) -> Response:
         dataset_url = f"histories/{history_id}/contents/{content_id}"
-        delete_response = self._delete(dataset_url, {"purge": purge, "stop_job": stop_job}, json=True)
+        if use_query_params:
+            delete_response = self._delete(f"{dataset_url}?purge={purge}&stop_job={stop_job}")
+        else:
+            delete_response = self._delete(dataset_url, {"purge": purge, "stop_job": stop_job}, json=True)
         delete_response.raise_for_status()
         if wait_for_purge and delete_response.status_code == 202:
             return self.wait_for_purge(history_id, content_id)

test/functional/tools/for_workflows/pileup.xml

@@ -3,7 +3,7 @@
 printf 'Summary' > '$out_file1'
     ]]></command>
     <inputs>
-        <param name="input1" type="data" format="bam" multiple="true" min="1" label="BAM Inputs">
+        <param name="input1" type="data" format="unsorted.bam" multiple="true" min="1" label="BAM Inputs">
             <validator check="bam_index" message="Metadata missing, click the pencil icon in the history item and use the auto-detect feature to correct this issue." type="metadata" />
         </param>
         <param name="reference" type="data" format="fasta" label="Fasta Input"/>

test/integration/test_extended_metadata.py

@@ -91,6 +91,37 @@ class ExtendedMetadataIntegrationTestCase(integration_util.IntegrationTestCase):
         assert dataset["created_from_basename"] == "4.bed"
 
 
+class ExtendedMetadataDeferredIntegrationTestCase(integration_util.IntegrationTestCase):
+    def setUp(self):
+        super().setUp()
+        self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        config["metadata_strategy"] = "extended"
+        config["object_store_store_by"] = "uuid"
+        config["retry_metadata_internally"] = False
+
+    def test_deferred_upload_with_metadata_files(self):
+        history_id = self.dataset_populator.new_history()
+        deferred_dataset = self.dataset_populator.create_deferred_hda(
+            history_id=history_id,
+            uri="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bam",
+            ext="bam",
+        )
+        inputs = {"input1": {"src": "hda", "id": deferred_dataset["id"]}}
+        run_response = self.dataset_populator.run_tool("cat1", inputs=inputs, history_id=history_id)
+        self.dataset_populator.wait_for_job(run_response["jobs"][0]["id"], assert_ok=True)
+        bam_dataset = self.dataset_populator.get_history_dataset_details(
+            history_id=history_id, content_id=run_response["outputs"][0]["id"]
+        )
+        assert bam_dataset["state"] == "ok"
+        assert bam_dataset["extension"] == "bam"
+        metadata_response = self._get(f"datasets/{bam_dataset['id']}/metadata_file?metadata_file=bam_index")
+        metadata_response.raise_for_status()
+        assert metadata_response.content.startswith(b"BAI")
+
+
 class ExtendedMetadataIntegrationInstance(integration_util.IntegrationInstance):
     """Describe a Galaxy test instance with embedded pulsar configured."""