From ecbce1369df5eb314d407f5470f8e1bde121d44d Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 27 May 2021 13:29:15 +0200 Subject: [PATCH 01/20] fix pysam.view call otherwise samtools output is the return value of `pysam.view` and the output stays empty. see also https://github.com/pysam-developers/pysam/issues/677 --- lib/galaxy/tool_util/verify/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index b3b1abda323..e59be9540db 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -184,12 +184,12 @@ def _bam_to_sam(local_name, temp_name): temp_local = tempfile.NamedTemporaryFile(suffix='.sam', prefix='local_bam_converted_to_sam_') with tempfile.NamedTemporaryFile(suffix='.sam', prefix='history_bam_converted_to_sam_', delete=False) as temp: try: - pysam.view('-h', '-o%s' % temp_local.name, local_name) + pysam.view('-h', '-o', temp_local.name, local_name, catch_stdout=False) except Exception as e: msg = "Converting local (test-data) BAM to SAM failed: %s" % unicodify(e) raise Exception(msg) try: - pysam.view('-h', '-o%s' % temp.name, temp_name) + pysam.view('-h', '-o', temp.name, temp_name, catch_stdout=False) except Exception as e: msg = "Converting history BAM to SAM failed: %s" % unicodify(e) raise Exception(msg) From 1cf34d74d063094343328d18adb8e4328e06a69e Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 27 May 2021 14:58:26 +0200 Subject: [PATCH 02/20] psyam should not add a `@PG` headerline i.e. leave the sam content of the bam file untouched --- lib/galaxy/tool_util/verify/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index e59be9540db..5414d31ac75 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -184,12 +184,12 @@ def _bam_to_sam(local_name, temp_name): temp_local = tempfile.NamedTemporaryFile(suffix='.sam', prefix='local_bam_converted_to_sam_') with tempfile.NamedTemporaryFile(suffix='.sam', prefix='history_bam_converted_to_sam_', delete=False) as temp: try: - pysam.view('-h', '-o', temp_local.name, local_name, catch_stdout=False) + pysam.view('-h', '--no-PG', '-o', temp_local.name, local_name, catch_stdout=False) except Exception as e: msg = "Converting local (test-data) BAM to SAM failed: %s" % unicodify(e) raise Exception(msg) try: - pysam.view('-h', '-o', temp.name, temp_name, catch_stdout=False) + pysam.view('-h', '--no-PG', '-o', temp.name, temp_name, catch_stdout=False) except Exception as e: msg = "Converting history BAM to SAM failed: %s" % unicodify(e) raise Exception(msg) From 01eba8ee7a509ca52d8d9846a9e8d44609851d86 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 27 May 2021 15:10:05 +0200 Subject: [PATCH 03/20] add ftype for sam_to_bam test to also convert this to sam in tests --- test/functional/tools/sam_to_bam.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/tools/sam_to_bam.xml b/test/functional/tools/sam_to_bam.xml index b01152a7670..3420ed73ec9 100644 --- a/test/functional/tools/sam_to_bam.xml +++ b/test/functional/tools/sam_to_bam.xml @@ -11,7 +11,7 @@ cat '$input1' > '$out_file1' - + From 0028824807636dd8ec7ea43c8518eaa8043a4af2 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Fri, 28 May 2021 11:15:02 -0400 Subject: [PATCH 04/20] Fix invocation time in workflow reports. --- lib/galaxy/managers/markdown_util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/managers/markdown_util.py b/lib/galaxy/managers/markdown_util.py index 278803c3fbb..64c596c0ac9 100644 --- a/lib/galaxy/managers/markdown_util.py +++ b/lib/galaxy/managers/markdown_util.py @@ -685,8 +685,8 @@ history_dataset_collection_display(input={}) return ("workflow_display(workflow_id=%s)\n" % invocation.workflow.stored_workflow.id, False) if container == "history_link": return ("history_link(history_id=%s)\n" % invocation.history.id, False) - if container == "invocation_date": - return ("invocation_date(invocation_id=%s)\n" % invocation.id, False) + if container == "invocation_date" or container == "invocation_time": + return ("invocation_time(invocation_id=%s)\n" % invocation.id, False) ref_object_type = None output_match = re.search(OUTPUT_LABEL_PATTERN, line) input_match = re.search(INPUT_LABEL_PATTERN, line) From e9fd8a8eb9fcb99be5a149af38580f8d0358bf75 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Fri, 28 May 2021 11:15:45 -0400 Subject: [PATCH 05/20] On second look, I don't think this was ever invocation_date? --- lib/galaxy/managers/markdown_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/managers/markdown_util.py b/lib/galaxy/managers/markdown_util.py index 64c596c0ac9..f3471e92f08 100644 --- a/lib/galaxy/managers/markdown_util.py +++ b/lib/galaxy/managers/markdown_util.py @@ -685,7 +685,7 @@ history_dataset_collection_display(input={}) return ("workflow_display(workflow_id=%s)\n" % invocation.workflow.stored_workflow.id, False) if container == "history_link": return ("history_link(history_id=%s)\n" % invocation.history.id, False) - if container == "invocation_date" or container == "invocation_time": + if container == "invocation_time": return ("invocation_time(invocation_id=%s)\n" % invocation.id, False) ref_object_type = None output_match = re.search(OUTPUT_LABEL_PATTERN, line) From cd1bea3434253d92b7d70727134e43a197e3fdad Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Fri, 28 May 2021 12:24:02 -0400 Subject: [PATCH 06/20] Reorganize VALID_ARGUMENTS to specify invocation_time as invocation flavored markdown --- lib/galaxy/managers/markdown_parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/managers/markdown_parse.py b/lib/galaxy/managers/markdown_parse.py index 3c4effd7f9a..8826e6481aa 100644 --- a/lib/galaxy/managers/markdown_parse.py +++ b/lib/galaxy/managers/markdown_parse.py @@ -33,9 +33,9 @@ VALID_ARGUMENTS = { "tool_stdout": ["step", "job_id"], "generate_galaxy_version": [], "generate_time": [], - "invocation_time": ["invocation_id"], "visualization": DYNAMIC_ARGUMENTS, # Invocation Flavored Markdown + "invocation_time": ["invocation_id"], "invocation_outputs": [], "invocation_inputs": [], } From 16d750be1368998bdd41d4ea6deb769fef248523 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Fri, 28 May 2021 12:33:12 -0400 Subject: [PATCH 07/20] Add test for invocation_time --- test/unit/workflows/test_workflow_markdown.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/unit/workflows/test_workflow_markdown.py b/test/unit/workflows/test_workflow_markdown.py index 742d557b451..262b78650c5 100644 --- a/test/unit/workflows/test_workflow_markdown.py +++ b/test/unit/workflows/test_workflow_markdown.py @@ -53,6 +53,18 @@ history_dataset_peek(input=input1) assert "```galaxy\nhistory_dataset_peek(history_dataset_id=567)\n```" in galaxy_markdown +def test_invocation_time(): + workflow_markdown = """ +And outputs... + +```galaxy +invocation_time() +``` +""" + galaxy_markdown = resolved_markdown(workflow_markdown) + assert "```galaxy\ninvocation_time(invocation_id=44)\n```" in galaxy_markdown + + def test_output_reference_mapping(): workflow_markdown = """ And outputs... @@ -78,6 +90,7 @@ def example_invocation(trans): invocation = model.WorkflowInvocation() workflow = yaml_to_model(TEST_WORKFLOW_YAML) workflow.id = 342 + invocation.id = 44 invocation.workflow = workflow # TODO: fix this to use workflow id and eliminate hack. From 8dd650c438f8fa3b4bc0ef7041f7ec1c5a332b3f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 31 May 2021 16:21:20 +0200 Subject: [PATCH 08/20] Test migrations against postgres 9.6 --- .github/workflows/db_indexes.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/db_indexes.yaml b/.github/workflows/db_indexes.yaml index b11d21bf025..64f46d667fa 100644 --- a/.github/workflows/db_indexes.yaml +++ b/.github/workflows/db_indexes.yaml @@ -11,9 +11,13 @@ jobs: matrix: python-version: ['3.7'] db: ['postgresql', 'sqlite'] + postgresql-version: ['13'] + include: + - db: postgresql + postgresql-version: '9.6' services: postgres: - image: postgres:13 + image: postgres:${{ matrix.postgresql-version }} env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres From e0b92245007943f3252ee670799c82924fbdc9b4 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 31 May 2021 16:21:59 +0200 Subject: [PATCH 09/20] Use PROCEDURE keyword on postgresql < 11 --- .../model/migrate/triggers/update_audit_table.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/model/migrate/triggers/update_audit_table.py b/lib/galaxy/model/migrate/triggers/update_audit_table.py index d3340f8b727..29bf9cfd76a 100644 --- a/lib/galaxy/model/migrate/triggers/update_audit_table.py +++ b/lib/galaxy/model/migrate/triggers/update_audit_table.py @@ -83,7 +83,7 @@ def _postgres_install(engine): $BODY$ """ - def statement_trigger_def(source_table, id_field, operation, when="AFTER"): + def statement_trigger_def(source_table, id_field, operation, when="AFTER", function_keyword="FUNCTION"): fn = f"{fn_prefix}_{id_field}" # Postgres supports many triggers per operation/table so the label can @@ -95,10 +95,10 @@ def _postgres_install(engine): CREATE TRIGGER {trigger_name} {when} {operation} ON {source_table} REFERENCING NEW TABLE AS new_table - FOR EACH STATEMENT EXECUTE FUNCTION {fn}(); + FOR EACH STATEMENT EXECUTE {function_keyword} {fn}(); """ - def row_trigger_def(source_table, id_field, operation, when="AFTER"): + def row_trigger_def(source_table, id_field, operation, when="AFTER", function_keyword="FUNCTION"): fn = f"{fn_prefix}_{id_field}" label = f"history_audit_by_{id_field}" @@ -107,20 +107,24 @@ def _postgres_install(engine): return f""" CREATE TRIGGER {trigger_name} {when} {operation} ON {source_table} - FOR EACH ROW EXECUTE FUNCTION {fn}(); + FOR EACH ROW EXECUTE {function_keyword} {fn}(); """ # pick row or statement triggers depending on postgres version version = engine.dialect.server_version_info[0] trigger_fn = statement_trigger_fn if version > 10 else row_trigger_fn trigger_def = statement_trigger_def if version > 10 else row_trigger_def + # In the syntax of CREATE TRIGGER, the keywords FUNCTION and PROCEDURE are equivalent, + # but the referenced function must in any case be a function, not a procedure. + # The use of the keyword PROCEDURE here is historical and deprecated (https://www.postgresql.org/docs/11/sql-createtrigger.html). + function_keyword = "FUNCTION" if version > 10 else "PROCEDURE" for id_field in ["history_id", "id"]: sql.append(trigger_fn(id_field)) for source_table, id_field in trigger_config.items(): for operation in ["UPDATE", "INSERT"]: - sql.append(trigger_def(source_table, id_field, operation)) + sql.append(trigger_def(source_table, id_field, operation, function_keyword=function_keyword)) return sql From e8a72b0f13e7c6b1e50726136dc7b3c00104f6f2 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Mon, 31 May 2021 18:58:45 +0200 Subject: [PATCH 10/20] return selectAll --- .../LibraryFolder/TopToolbar/add-datasets.js | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/client/src/components/Libraries/LibraryFolder/TopToolbar/add-datasets.js b/client/src/components/Libraries/LibraryFolder/TopToolbar/add-datasets.js index 9f3a2cc977c..79cb6ae1b93 100644 --- a/client/src/components/Libraries/LibraryFolder/TopToolbar/add-datasets.js +++ b/client/src/components/Libraries/LibraryFolder/TopToolbar/add-datasets.js @@ -710,6 +710,26 @@ var AddDatasets = Backbone.View.extend({ } }, + /** + * User clicked the checkbox in the table heading + * @param {context} event + */ + selectAll: function (event) { + var selected = event.target.checked; + var self = this; + // Iterate each checkbox + $(":checkbox", "#dataset_list tbody").each(function () { + this.checked = selected; + var $row = $(this).closest("tr"); + // Change color of selected/unselected + if (selected) { + self.makeDarkRow($row); + } else { + self.makeWhiteRow($row); + } + }); + }, + makeDarkRow: function ($row) { $row.addClass("table-primary"); }, From e58eaa04029954b287bd024063b61cb827256840 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Tue, 1 Jun 2021 15:14:57 +0200 Subject: [PATCH 11/20] fix https://github.com/galaxyproject/galaxy/issues/11916 --- .../LibraryFolder/TopToolbar/FolderTopBar.vue | 2 +- .../Libraries/LibraryFolder/services.js | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue b/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue index a9fddc56612..e194e618ba2 100644 --- a/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue +++ b/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue @@ -271,7 +271,7 @@ export default { async getSelected() { if (this.isAllSelectedMode) { this.$emit("setBusy", true); - const selected = await this.services.getFilteredFolderContents(this.folder_id, this.unselected); + const selected = await this.services.getFilteredFolderContents(this.folder_id, this.unselected, this.$parent.search_text); this.$emit("setBusy", false); return selected; } else { diff --git a/client/src/components/Libraries/LibraryFolder/services.js b/client/src/components/Libraries/LibraryFolder/services.js index 4225df9be57..34f3f0e47ee 100644 --- a/client/src/components/Libraries/LibraryFolder/services.js +++ b/client/src/components/Libraries/LibraryFolder/services.js @@ -8,8 +8,11 @@ export class Services { } async getFolderContents(id, include_deleted, limit, offset, search_text = false) { - search_text = search_text ? `&search_text=${encodeURI(search_text.trim())}` : ""; - const url = `${this.root}api/folders/${id}/contents?include_deleted=${include_deleted}&limit=${limit}&offset=${offset}${search_text}`; + const url = `${ + this.root + }api/folders/${id}/contents?include_deleted=${include_deleted}&limit=${limit}&offset=${offset}${this.getSearchQuery( + search_text + )}`; try { const response = await axios.get(url); return response.data; @@ -18,13 +21,17 @@ export class Services { } } - async getFilteredFolderContents(id, excluded) { - const contents = await axios.get(`${this.root}api/folders/${id}/contents`); + async getFilteredFolderContents(id, excluded, search_text) { + const contents = await axios.get(`${this.root}api/folders/${id}/contents?${this.getSearchQuery(search_text)}`); return contents.data.folder_contents.filter((item) => { return !excluded.some((exc) => exc.id === item.id); }); } + getSearchQuery(search_text) { + return search_text ? `&search_text=${encodeURI(search_text.trim())}` : ""; + } + updateFolder(item, onSucess, onError) { const url = `${this.root}api/folders/${item.id}`; try { From a78e99bca6a8a36e1a9498249213ac71a20d7671 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Tue, 1 Jun 2021 14:33:51 +0200 Subject: [PATCH 12/20] fix https://github.com/galaxyproject/galaxy/issues/12014 --- .../LibraryFolder/TopToolbar/FolderTopBar.vue | 1 - .../import-to-history/import-collection.js | 101 +++++++----------- 2 files changed, 41 insertions(+), 61 deletions(-) diff --git a/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue b/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue index a9fddc56612..4abeddcf958 100644 --- a/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue +++ b/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue @@ -319,7 +319,6 @@ export default { if (isCollection) { new mod_import_collection.ImportCollectionModal({ selected: checkedItems, - allDatasets: this.allDatasets, }); } else { new mod_import_dataset.ImportDatasetModal({ diff --git a/client/src/components/Libraries/LibraryFolder/TopToolbar/import-to-history/import-collection.js b/client/src/components/Libraries/LibraryFolder/TopToolbar/import-to-history/import-collection.js index b1325b50c9c..6529dc50e7b 100644 --- a/client/src/components/Libraries/LibraryFolder/TopToolbar/import-to-history/import-collection.js +++ b/client/src/components/Libraries/LibraryFolder/TopToolbar/import-to-history/import-collection.js @@ -42,36 +42,40 @@ var ImportCollectionModal = Backbone.View.extend({ showCollectionSelect: function (e) { const Galaxy = getGalaxyInstance(); var checked_items = this.findCheckedItems(); - var template = this.templateCollectionSelectModal(); + if (checked_items.length === 0) { + Toast.info("You must select some datasets first."); + } else { + var template = this.templateCollectionSelectModal(); - var promise = this.fetchUserHistories(); - promise - .done(() => { - this.modal = Galaxy.modal; - this.modal.show({ - closing_events: true, - title: "Create History Collection from Datasets", - body: template({ - selected_datasets: checked_items.dataset_ids.length, - histories: this.histories.models, - }), - buttons: { - Continue: () => { - this.showCollectionBuilder(checked_items.dataset_ids); + var promise = this.fetchUserHistories(); + promise + .done(() => { + this.modal = Galaxy.modal; + this.modal.show({ + closing_events: true, + title: "Create History Collection from Datasets", + body: template({ + selected_datasets: checked_items.dataset_ids.length, + histories: this.histories.models, + }), + buttons: { + Continue: () => { + this.showCollectionBuilder(checked_items.dataset_ids); + }, + Close: () => { + Galaxy.modal.hide(); + }, }, - Close: () => { - Galaxy.modal.hide(); - }, - }, + }); + }) + .fail((model, response) => { + if (typeof response.responseJSON !== "undefined") { + Toast.error(response.responseJSON.err_msg); + } else { + Toast.error("An error occurred."); + } }); - }) - .fail((model, response) => { - if (typeof response.responseJSON !== "undefined") { - Toast.error(response.responseJSON.err_msg); - } else { - Toast.error("An error occurred."); - } - }); + } }, /** * Note: The collection creation process expects ldda_ids as ids @@ -79,21 +83,17 @@ var ImportCollectionModal = Backbone.View.extend({ * The code below overwrites the id with ldda_id for this reason. */ showCollectionBuilder: function (checked_items) { - let collection_elements = []; - const elements_source = this.modal.$('input[type="radio"]:checked').val(); - if (elements_source === "selection") { - for (let i = checked_items.length - 1; i >= 0; i--) { - const collection_item = {}; - const dataset = checked_items[i]; - collection_item.id = dataset.ldda_id; - collection_item.name = dataset.name; - collection_item.deleted = dataset.deleted; - collection_item.state = dataset.state; - collection_elements.push(collection_item); - } - } else if (elements_source === "folder") { - collection_elements = this.options.allDatasets; + const collection_elements = []; + for (let i = checked_items.length - 1; i >= 0; i--) { + const collection_item = {}; + const dataset = checked_items[i]; + collection_item.id = dataset.ldda_id; + collection_item.name = dataset.name; + collection_item.deleted = dataset.deleted; + collection_item.state = dataset.state; + collection_elements.push(collection_item); } + const new_history_name = this.modal.$("input[name=history_name]").val(); if (new_history_name !== "") { this.createNewHistory(new_history_name) @@ -198,25 +198,6 @@ var ImportCollectionModal = Backbone.View.extend({ templateCollectionSelectModal: function () { return _.template( `
-
-

Which datasets?

-
-
- disabled <% } else { %> checked <% } %> /> - -
-
- checked <% } %> > - -
-
-

Collection type

From 126909be0419826128e84c42aeb005760370abab Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Tue, 1 Jun 2021 14:35:50 +0200 Subject: [PATCH 13/20] remove unused function --- .../Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue b/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue index 4abeddcf958..81c10136802 100644 --- a/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue +++ b/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue @@ -249,10 +249,7 @@ export default { const Galaxy = getGalaxyInstance(); // logic from legacy code return !!(this.contains_file_or_folder && Galaxy.user); - }, - allDatasets: function () { - return this.folderContents.filter((element) => element.type === "file"); - }, + } }, methods: { updateSearch: function (value) { From 3ba6230c3ac13135e15ed0f8fd2a553716bb924c Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Tue, 1 Jun 2021 15:30:00 +0200 Subject: [PATCH 14/20] fix lint --- .../Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue b/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue index e194e618ba2..3663690071b 100644 --- a/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue +++ b/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue @@ -271,7 +271,11 @@ export default { async getSelected() { if (this.isAllSelectedMode) { this.$emit("setBusy", true); - const selected = await this.services.getFilteredFolderContents(this.folder_id, this.unselected, this.$parent.search_text); + const selected = await this.services.getFilteredFolderContents( + this.folder_id, + this.unselected, + this.$parent.search_text + ); this.$emit("setBusy", false); return selected; } else { From 637109b22298827e5ae1ca2fe2cdf21a86c234c9 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Tue, 1 Jun 2021 15:51:01 +0200 Subject: [PATCH 15/20] lint --- .../Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue b/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue index 81c10136802..5ef875f7234 100644 --- a/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue +++ b/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue @@ -249,7 +249,7 @@ export default { const Galaxy = getGalaxyInstance(); // logic from legacy code return !!(this.contains_file_or_folder && Galaxy.user); - } + }, }, methods: { updateSearch: function (value) { From bd6fe02aa84033967d6ae20b832d66ce9a54cd76 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 1 Jun 2021 17:45:12 +0200 Subject: [PATCH 16/20] Fix set metadata for primary discovered outputs We were previously setting metadata on the path that `filename_override` points to, which (should be an output in the job working directory, or an output in the object store. That does not work for `unnnamed_outputs` that are inferred via galaxy.json and that are discovered using collect_dynamic_outputs. In most cases that doesn't lead to an exception, but it does fail for bam files (and a small subset of other datatypes I would guess) if you specify the extension explicitly (if the extension is not set, sniffing on the empty dataset results in `data`, which will prevent the exception). For datatypes that don't fail in `set_metadata` on empty input files metadata would then be set again when collecting dynamic outputs, this time using the correct path. The fix here is to check whether the file we're setting metadata for is actually an unnamed output, and override the path with the path to the file in the working directory. To prevent setting metadata twice we also skip setting metadata on files that already have a database id (which should only be these primary discovered outputs). --- lib/galaxy/metadata/set_metadata.py | 41 ++++++++++++++++++----------- lib/galaxy/model/store/discover.py | 3 ++- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py index f17bd01a3ce..48010bcd561 100644 --- a/lib/galaxy/metadata/set_metadata.py +++ b/lib/galaxy/metadata/set_metadata.py @@ -139,6 +139,7 @@ def set_metadata_portable(): version_string = "" export_store = None + final_job_state = 'ok' if extended_metadata_collection: tool_dict = metadata_params["tool"] stdio_exit_code_dicts, stdio_regex_dicts = tool_dict["stdio_exit_codes"], tool_dict["stdio_regexes"] @@ -185,7 +186,7 @@ def set_metadata_portable(): if os.path.exists(COMMAND_VERSION_FILENAME): version_string = open(COMMAND_VERSION_FILENAME).read() - job_context = ExpressionContext(dict(stdout=tool_stdout, stderr=tool_stderr)) + expression_context = ExpressionContext(dict(stdout=tool_stdout, stderr=tool_stderr)) # Load outputs. export_store = store.DirectoryModelExportStore('metadata/outputs_populated', serialize_dataset_objects=True, for_edit=True, strip_metadata_files=False, serialize_jobs=False) @@ -195,6 +196,27 @@ def set_metadata_portable(): # Remove in 21.09, this should only happen for jobs that started on <= 20.09 and finish now import_model_store = None + job_context = SessionlessJobContext( + metadata_params, + tool_provided_metadata, + object_store, + export_store, + import_model_store, + os.path.join(tool_job_working_directory, "working"), + final_job_state=final_job_state, + ) + + unnamed_id_to_path = {} + for unnamed_output_dict in job_context.tool_provided_metadata.get_unnamed_outputs(): + destination = unnamed_output_dict["destination"] + elements = unnamed_output_dict["elements"] + destination_type = destination["type"] + if destination_type == 'hdas': + for element in elements: + filename = element.get('filename') + if filename: + unnamed_id_to_path[element['object_id']] = os.path.join(job_context.job_working_directory, filename) + for output_name, output_dict in outputs.items(): dataset_instance_id = output_dict["id"] klass = getattr(galaxy.model, output_dict.get('model_class', 'HistoryDatasetAssociation')) @@ -219,7 +241,7 @@ def set_metadata_portable(): # Same block as below... set_meta_kwds = stringify_dictionary_keys(json.load(open(filename_kwds))) # load kwds; need to ensure our keywords are not unicode try: - dataset.dataset.external_filename = dataset_filename_override + dataset.dataset.external_filename = unnamed_id_to_path.get(dataset_instance_id, dataset_filename_override) store_by = output_dict.get("object_store_store_by", legacy_object_store_store_by) extra_files_dir_name = "dataset_%s_files" % getattr(dataset.dataset, store_by) files_path = os.path.abspath(os.path.join(tool_job_working_directory, "working", extra_files_dir_name)) @@ -240,9 +262,9 @@ def set_metadata_portable(): if extended_metadata_collection: meta = tool_provided_metadata.get_dataset_meta(output_name, dataset.dataset.id, dataset.dataset.uuid) if meta: - context = ExpressionContext(meta, job_context) + context = ExpressionContext(meta, expression_context) else: - context = job_context + context = expression_context # Lazy and unattached # if getattr(dataset, "hidden_beneath_collection_instance", None): @@ -301,17 +323,6 @@ def set_metadata_portable(): if extended_metadata_collection: # discover extra outputs... - - job_context = SessionlessJobContext( - metadata_params, - tool_provided_metadata, - object_store, - export_store, - import_model_store, - os.path.join(tool_job_working_directory, "working"), - final_job_state=final_job_state, - ) - output_collections = {} for name, output_collection in metadata_params["output_collections"].items(): output_collections[name] = import_model_store.sa_session.query(HistoryDatasetCollectionAssociation).find(output_collection["id"]) diff --git a/lib/galaxy/model/store/discover.py b/lib/galaxy/model/store/discover.py index 07ea899ce20..5771c200a20 100644 --- a/lib/galaxy/model/store/discover.py +++ b/lib/galaxy/model/store/discover.py @@ -171,7 +171,8 @@ class ModelPersistenceContext(metaclass=abc.ABCMeta): if info is not None: primary_data.info = info - if filename: + if filename and not primary_data.id: + # If primary data has an id it should already have gone through the metadata process self.set_datasets_metadata(datasets=[primary_data], datasets_attributes=[dataset_attributes]) return primary_data From 82d66bdd2980e96327a5f3643a505e34a00890d0 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 2 Jun 2021 11:47:31 +0200 Subject: [PATCH 17/20] That's needed for things set via galaxy.json But it does mean that we might be running set_meta twice. Need to figure out a better way of doing this. --- lib/galaxy/model/store/discover.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/galaxy/model/store/discover.py b/lib/galaxy/model/store/discover.py index 5771c200a20..07ea899ce20 100644 --- a/lib/galaxy/model/store/discover.py +++ b/lib/galaxy/model/store/discover.py @@ -171,8 +171,7 @@ class ModelPersistenceContext(metaclass=abc.ABCMeta): if info is not None: primary_data.info = info - if filename and not primary_data.id: - # If primary data has an id it should already have gone through the metadata process + if filename: self.set_datasets_metadata(datasets=[primary_data], datasets_attributes=[dataset_attributes]) return primary_data From 5b5ebf90d5be2cb88ceb1da5c783d1a095db82c1 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 2 Jun 2021 18:34:54 +0200 Subject: [PATCH 18/20] Add API test for fetch_data with bam fails This fails without the preceding commit. --- lib/galaxy_test/api/test_tools_upload.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/galaxy_test/api/test_tools_upload.py b/lib/galaxy_test/api/test_tools_upload.py index 824b381361d..4acaac47690 100644 --- a/lib/galaxy_test/api/test_tools_upload.py +++ b/lib/galaxy_test/api/test_tools_upload.py @@ -322,6 +322,28 @@ class ToolsUploadTestCase(ApiTestCase): assert output0["state"] == "ok" assert output1["state"] == "error" + @uses_test_history(require_new=False) + def test_fetch_bam_file_from_url_with_extension_set(self, history_id): + destination = {"type": "hdas"} + targets = [{ + "destination": destination, + "items": [ + { + "src": "url", + "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bam", + "ext": "bam" + }, + ] + }] + payload = { + "history_id": history_id, + "targets": json.dumps(targets), + } + fetch_response = self.dataset_populator.fetch(payload) + self._assert_status_code_is(fetch_response, 200) + outputs = fetch_response.json()["outputs"] + self.dataset_populator.get_history_dataset_details(history_id, dataset=outputs[0], assert_ok=True) + @skip_without_datatype("velvet") def test_composite_datatype(self): with self.dataset_populator.test_history() as history_id: From c16446b9a313f921092d71c7570f08c524584243 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 2 Jun 2021 18:36:23 +0200 Subject: [PATCH 19/20] Use Job.states.OK --- lib/galaxy/metadata/set_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py index 48010bcd561..dea9b931275 100644 --- a/lib/galaxy/metadata/set_metadata.py +++ b/lib/galaxy/metadata/set_metadata.py @@ -139,7 +139,7 @@ def set_metadata_portable(): version_string = "" export_store = None - final_job_state = 'ok' + final_job_state = Job.states.OK if extended_metadata_collection: tool_dict = metadata_params["tool"] stdio_exit_code_dicts, stdio_regex_dicts = tool_dict["stdio_exit_codes"], tool_dict["stdio_regexes"] From bd5596f3828a419a1c4b44fdc7ea0ba41b813985 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 2 Jun 2021 16:30:58 +0200 Subject: [PATCH 20/20] Maybe skip setting metadata twice --- lib/galaxy/metadata/set_metadata.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py index dea9b931275..1d25c32de58 100644 --- a/lib/galaxy/metadata/set_metadata.py +++ b/lib/galaxy/metadata/set_metadata.py @@ -257,7 +257,10 @@ def set_metadata_portable(): setattr(dataset.metadata, metadata_name, metadata_file_override) if output_dict.get("validate", False): set_validated_state(dataset) - set_meta(dataset, file_dict) + if dataset_instance_id not in unnamed_id_to_path: + # We're going to run through set_metadata in collect_dynamic_outputs with more contextual metadata, + # so skip set_meta here. + set_meta(dataset, file_dict) if extended_metadata_collection: meta = tool_provided_metadata.get_dataset_meta(output_name, dataset.dataset.id, dataset.dataset.uuid)