From 471162eadb4408318db534c6d24f7685bedb29eb Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 11 Sep 2018 14:01:36 -0400 Subject: [PATCH 01/17] [18.09] Fix new extract dataset to perserve collection element identifier. --- lib/galaxy/model/__init__.py | 15 +++++++++++++-- lib/galaxy/tools/__init__.py | 9 +++++---- lib/galaxy/tools/extract_dataset.xml | 12 +++++++++--- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index ed2c75ded0f..0724ed1bf39 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2599,7 +2599,7 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot self.version = self.version + 1 if self.version else 1 session.add(past_hda) - def copy(self, parent_id=None, copy_tags=None, force_flush=True, copy_hid=True): + def copy(self, parent_id=None, copy_tags=None, force_flush=True, copy_hid=True, new_name=None): """ Create a copy of this HDA. """ @@ -2607,7 +2607,7 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot if copy_hid: hid = self.hid hda = HistoryDatasetAssociation(hid=hid, - name=self.name, + name=new_name or self.name, info=self.info, blurb=self.blurb, peek=self.peek, @@ -3451,6 +3451,17 @@ class DatasetCollection(Dictifiable, UsesAnnotations): elements.append(element) return elements + @property + def first_dataset_element(self): + for element in self.elements: + if element.is_collection: + first_element = element.child_collection.first_dataset_element + if first_element: + return first_element + else: + return element + return None + @property def state(self): # TODO: DatasetCollection state handling... diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index bf83a5d640e..ef06bf71a49 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2481,14 +2481,15 @@ class ExtractDatasetCollectionTool(DatabaseOperationTool): assert collection_type in ["list", "paired"] how = incoming["which"]["which_dataset"] if how == "first": - extracted = collection.dataset_instances[0] + extracted_element = collection.first_dataset_element elif how == "by_identifier": - extracted = collection[incoming["which"]["identifier"]].element_object + extracted_element = collection[incoming["which"]["identifier"]] elif how == "by_index": - extracted = collection[int(incoming["which"]["index"])].element_object + extracted_element = collection[int(incoming["which"]["index"])] else: raise Exception("Invalid tool parameters.") - extracted_o = extracted.copy(copy_tags=tags) + extracted = extracted_element.element_object + extracted_o = extracted.copy(copy_tags=tags, new_name=extracted_element.element_identifier) self._add_datasets_to_history(history, [extracted_o]) out_data["output"] = extracted_o diff --git a/lib/galaxy/tools/extract_dataset.xml b/lib/galaxy/tools/extract_dataset.xml index 368f3df3d59..f4dd795e8f2 100644 --- a/lib/galaxy/tools/extract_dataset.xml +++ b/lib/galaxy/tools/extract_dataset.xml @@ -50,7 +50,9 @@ This tool will create new history datasets from your collection but your quota u - + + + @@ -63,7 +65,9 @@ This tool will create new history datasets from your collection but your quota u - + + + @@ -76,7 +80,9 @@ This tool will create new history datasets from your collection but your quota u - + + + From ba4e7d199b4bc080dcd741710d2241147bdbe745 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 12 Sep 2018 08:12:52 +0200 Subject: [PATCH 02/17] Make gro and top datatypes tabular subclasses I think that was the intention in https://github.com/galaxyproject/galaxy/commit/56d55399ebe2ecd614a7ba5a146739f58e917541 --- config/datatypes_conf.xml.sample | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index e11a7f2d435..3c7d183d191 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -576,10 +576,10 @@ - + - + From 1f8c234c2be8baac9db94e17f4727e6c3c6c55ef Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Wed, 12 Sep 2018 09:03:16 -0400 Subject: [PATCH 03/17] Make top datatype inherit from Text. --- config/datatypes_conf.xml.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 3c7d183d191..bf49f95419a 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -576,7 +576,7 @@ - + From b4b49dc2028da2064fd34d7345b23b47150a1039 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 13 Sep 2018 12:52:20 -0400 Subject: [PATCH 04/17] Add self help to error reporting interface. --- .../scripts/mvc/dataset/dataset-error.js | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/client/galaxy/scripts/mvc/dataset/dataset-error.js b/client/galaxy/scripts/mvc/dataset/dataset-error.js index 74f735eab7e..9d14d8ea917 100644 --- a/client/galaxy/scripts/mvc/dataset/dataset-error.js +++ b/client/galaxy/scripts/mvc/dataset/dataset-error.js @@ -2,6 +2,12 @@ import _l from "utils/localization"; import Utils from "utils/utils"; import Ui from "mvc/ui/ui-misc"; import Form from "mvc/form/form-view"; +import Backbone from "backbone"; +import * as _ from "underscore"; + +/* global Galaxy */ +/* global $ */ + /** Dataset edit attributes view */ var View = Backbone.View.extend({ initialize: function() { @@ -57,12 +63,23 @@ var View = Backbone.View.extend({

An error occured while running the tool ${job.tool_id}.

Tool execution generated the following messages:

${_.escape(job.stderr)}
-

Report This Error

+ +

Troubleshoot This Error

- Usually the local Galaxy administrators regularly review errors that occur on the server - However, if you would like to provide additional information (such as what you were trying - to do when the error occurred) and a contact e-mail address, we will be better able to - investigate your problem and get back to you. + There are a number of help resources to self diagnose and + correct problems. + Start here: My job ended with an error. What can I do? +

+ +

Report This Error

+

+ Usually the local Galaxy administrators regularly review errors + that occur on the server However, if you would like to provide + additional information (such as what you were trying to do when + the error occurred) and a contact e-mail address, we will be + better able to investigate your problem and get back to you.

`); this.$el.append(this._getBugFormTemplate(dataset, job)); }, From 88bf3435775e94217bccfc46a5215311dea321aa Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 13 Sep 2018 14:25:59 -0400 Subject: [PATCH 05/17] Import Backbone consistently across the app --- client/galaxy/scripts/mvc/dataset/dataset-error.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/galaxy/scripts/mvc/dataset/dataset-error.js b/client/galaxy/scripts/mvc/dataset/dataset-error.js index 9d14d8ea917..b64694532c6 100644 --- a/client/galaxy/scripts/mvc/dataset/dataset-error.js +++ b/client/galaxy/scripts/mvc/dataset/dataset-error.js @@ -2,7 +2,7 @@ import _l from "utils/localization"; import Utils from "utils/utils"; import Ui from "mvc/ui/ui-misc"; import Form from "mvc/form/form-view"; -import Backbone from "backbone"; +import * as Backbone from "backbone"; import * as _ from "underscore"; /* global Galaxy */ From 84d66bea6008dc534aa6a2331e824005b204da19 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Fri, 14 Sep 2018 10:23:34 -0400 Subject: [PATCH 06/17] h2!=h3 --- client/galaxy/scripts/mvc/dataset/dataset-error.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/galaxy/scripts/mvc/dataset/dataset-error.js b/client/galaxy/scripts/mvc/dataset/dataset-error.js index b64694532c6..6a288277b51 100644 --- a/client/galaxy/scripts/mvc/dataset/dataset-error.js +++ b/client/galaxy/scripts/mvc/dataset/dataset-error.js @@ -73,7 +73,7 @@ var View = Backbone.View.extend({ target="_blank"> My job ended with an error. What can I do?

-

Report This Error

+

Report This Error

Usually the local Galaxy administrators regularly review errors that occur on the server However, if you would like to provide From df4fbe6b92457fa910f014bbd3a8dd84c25d3969 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Mon, 17 Sep 2018 11:22:51 -0400 Subject: [PATCH 07/17] Swap badge-default to badge-primary --- client/galaxy/scripts/mvc/tool/tools.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/galaxy/scripts/mvc/tool/tools.js b/client/galaxy/scripts/mvc/tool/tools.js index eca42113979..bd50d0c2a79 100644 --- a/client/galaxy/scripts/mvc/tool/tools.js +++ b/client/galaxy/scripts/mvc/tool/tools.js @@ -828,7 +828,7 @@ var templates = { '', '', "<% _.each( labels, function( label ){ %>", - '', + '', "<%- label %>", "", "<% }); %>", From b2193bddc2aea250a189171ae61ed992b0449c84 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Mon, 17 Sep 2018 11:23:51 -0400 Subject: [PATCH 08/17] Drop badge-default from docs, correctly refer to secondary. --- client/galaxy/docs/bootstrap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/galaxy/docs/bootstrap.md b/client/galaxy/docs/bootstrap.md index e44d6423e84..ec879739331 100644 --- a/client/galaxy/docs/bootstrap.md +++ b/client/galaxy/docs/bootstrap.md @@ -135,8 +135,8 @@ Disabled ```

- Default Primary + Secondary Success Warning Danger From a9273218424c1687bedf112591f2bbf735d32aa8 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Mon, 17 Sep 2018 11:25:28 -0400 Subject: [PATCH 09/17] Client format. --- client/galaxy/scripts/galaxy.pages.js | 6 ++- .../galaxy/scripts/mvc/tool/tool-form-base.js | 2 +- .../scripts/mvc/tool/tool-form-composite.js | 4 +- .../scripts/mvc/workflow/workflow-manager.js | 2 +- .../scripts/mvc/workflow/workflow-view.js | 43 +++++++++++-------- 5 files changed, 34 insertions(+), 23 deletions(-) diff --git a/client/galaxy/scripts/galaxy.pages.js b/client/galaxy/scripts/galaxy.pages.js index 2144f436268..a8bb8b9ccc4 100644 --- a/client/galaxy/scripts/galaxy.pages.js +++ b/client/galaxy/scripts/galaxy.pages.js @@ -447,7 +447,11 @@ WYMeditor.editor.prototype.dialog = function(dialogType, dialogFeatures, bodyHtm // Get item ID and name. var item_id = elt.val(); // Use ':first' because there are many labels in table; the first one is the item name. - var item_name = elt.closest('td').next('td').find('label').text(); + var item_name = elt + .closest("td") + .next("td") + .find("label") + .text(); if (make_importable) make_item_importable(item_info.controller, item_id, item_info.singular); diff --git a/client/galaxy/scripts/mvc/tool/tool-form-base.js b/client/galaxy/scripts/mvc/tool/tool-form-base.js index 5fac45cdf24..9239281cbfe 100644 --- a/client/galaxy/scripts/mvc/tool/tool-form-base.js +++ b/client/galaxy/scripts/mvc/tool/tool-form-base.js @@ -88,7 +88,7 @@ export default FormBase.extend({ } options.tool_errors && this.message.update({ - status: 'danger', + status: "danger", message: options.tool_errors, persistent: true }); diff --git a/client/galaxy/scripts/mvc/tool/tool-form-composite.js b/client/galaxy/scripts/mvc/tool/tool-form-composite.js index 3ce17a5595a..0a96cd2b641 100644 --- a/client/galaxy/scripts/mvc/tool/tool-form-composite.js +++ b/client/galaxy/scripts/mvc/tool/tool-form-composite.js @@ -129,14 +129,14 @@ var View = Backbone.View.extend({ this.wp_inputs = {}; function _ensureWorkflowParameter(wp_name) { - return self.wp_inputs[wp_name] = self.wp_inputs[wp_name] || { + return (self.wp_inputs[wp_name] = self.wp_inputs[wp_name] || { label: wp_name, name: wp_name, type: "text", color: `hsl( ${++wp_count * 100}, 70%, 30% )`, style: "ui-form-wp-source", links: [] - }; + }); } function _handleWorkflowParameter(value, callback) { diff --git a/client/galaxy/scripts/mvc/workflow/workflow-manager.js b/client/galaxy/scripts/mvc/workflow/workflow-manager.js index f247d2606be..826caf8f78e 100644 --- a/client/galaxy/scripts/mvc/workflow/workflow-manager.js +++ b/client/galaxy/scripts/mvc/workflow/workflow-manager.js @@ -278,7 +278,7 @@ class Workflow { $.each(node.output_terminals, (ot_id, ot) => { if (node.post_job_actions[`HideDatasetAction${ot.name}`] === undefined) { node.addWorkflowOutput(ot.name); - var callout = $(node.element).find(`.callout.${ot.name.replace(/(?=[()])/g, '\\')}`); + var callout = $(node.element).find(`.callout.${ot.name.replace(/(?=[()])/g, "\\")}`); callout.find("img").attr("src", `${Galaxy.root}static/images/fugue/asterisk-small.png`); wf.has_changes = true; } diff --git a/client/galaxy/scripts/mvc/workflow/workflow-view.js b/client/galaxy/scripts/mvc/workflow/workflow-view.js index 406d8dee70a..65645f3cfa4 100644 --- a/client/galaxy/scripts/mvc/workflow/workflow-view.js +++ b/client/galaxy/scripts/mvc/workflow/workflow-view.js @@ -278,46 +278,51 @@ export default Backbone.View.extend({ let workflow_versions = JSON.parse( $.ajax({ url: `${Galaxy.root}api/workflows/${self.options.id}/versions`, - async: false, + async: false }).responseText ); for (let i = 0; i < workflow_versions.length; i++) { let current_wf = workflow_versions[i]; - let version_text = `Version ${current_wf['version']}, ${current_wf['steps']} steps`; + let version_text = `Version ${current_wf["version"]}, ${current_wf["steps"]} steps`; let selected = false; if (i == self.workflow.workflow_version) { version_text = `${version_text} (active)`; selected = true; - }; + } _workflow_version_dropdown[version_text] = { version: i, - selected: selected, - } + selected: selected + }; } - return _workflow_version_dropdown - } + return _workflow_version_dropdown; + }; this.build_version_select = function() { let versions = this.get_workflow_versions(); $("#workflow-version-switch").empty(); $.each(versions, function(k, v) { - $('#workflow-version-switch').append($('').html(k).val(v.version).selected(v.selected)); + $("#workflow-version-switch").append( + $("") + .html(k) + .val(v.version) + .selected(v.selected) + ); }); - $("#workflow-version-switch").on('change', function () { - $('#workflow-version-switch').unbind('change'); + $("#workflow-version-switch").on("change", function() { + $("#workflow-version-switch").unbind("change"); if (this.value != self.workflow.workflow_version) { if (self.workflow && self.workflow.has_changes) { let r = confirm("There are unsaved changes to your workflow which will be lost. Continue ?"); if (r == false) { // We rebuild the version select list, to reset the selected version self.build_version_select(); - return + return; } } self.load_workflow(self.options.id, this.value); } - }) + }); }; this.load_workflow = function load_workflow(id, version) { @@ -360,19 +365,21 @@ export default Backbone.View.extend({ self.showWorkflowParameters(); }, error: function(response) { - window.show_modal("Loading workflow failed.", response.err_msg, { Ok: function( response ){ - window.onbeforeunload = undefined; - window.document.location = workflow_index; -}}) + window.show_modal("Loading workflow failed.", response.err_msg, { + Ok: function(response) { + window.onbeforeunload = undefined; + window.document.location = workflow_index; + } + }); }, beforeSubmit: function(data) { show_message("Loading workflow", "progress"); } }); - } + }; // Load workflow definition - this.load_workflow(self.options.id, self.options.version) + this.load_workflow(self.options.id, self.options.version); if (window.make_popupmenu) { make_popupmenu($("#workflow-options-button"), { Save: save_current_workflow, From 072d40f3e9a9050f034f440b76c8b84536bc0ead Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Mon, 17 Sep 2018 13:52:12 -0400 Subject: [PATCH 10/17] remove html tags and newlines from the help text this should improve the toolsearch for certain keywords and also make the search index smaller --- lib/galaxy/tools/search/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/search/__init__.py b/lib/galaxy/tools/search/__init__.py index fe59c89d0d0..71a605d436e 100644 --- a/lib/galaxy/tools/search/__init__.py +++ b/lib/galaxy/tools/search/__init__.py @@ -25,6 +25,7 @@ from galaxy.util import ExecutionTimer from galaxy.web.framework.helpers import to_unicode log = logging.getLogger(__name__) +HTML_CLEANER = re.compile("<[^<]+?>") class ToolBoxSearch(object): @@ -49,7 +50,7 @@ class ToolBoxSearch(object): self.storage, self.index = self._index_setup() # We keep track of how many times the tool index has been rebuilt. # We start at -1, so that after the first index the count is at 0, - # which is the same is the toolbox reload count. This way we can skip + # which is the same as the toolbox reload count. This way we can skip # reindexing if the index count is equal to the toolbox reload count. self.index_count = -1 @@ -105,7 +106,9 @@ class ToolBoxSearch(object): add_doc_kwds['labels'] = to_unicode(" ".join(tool.labels)) if index_help and tool.help: try: - add_doc_kwds['help'] = to_unicode(tool.help.render(host_url="", static_path="")) + raw_html = tool.help.render(host_url="", static_path="") + cleantext = re.sub(HTML_CLEANER, '', raw_html).replace('\n', '') + add_doc_kwds['help'] = to_unicode(cleantext) except Exception: # Don't fail to build index just because a help message # won't render. From b7f16228575215c9be134d1bbc1cf9e37b700488 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Mon, 17 Sep 2018 13:58:06 -0400 Subject: [PATCH 11/17] replace the replacement for newline from empty string to space, that should avoid unwanted word merges --- lib/galaxy/tools/search/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/search/__init__.py b/lib/galaxy/tools/search/__init__.py index 71a605d436e..19d9fa46e09 100644 --- a/lib/galaxy/tools/search/__init__.py +++ b/lib/galaxy/tools/search/__init__.py @@ -107,7 +107,7 @@ class ToolBoxSearch(object): if index_help and tool.help: try: raw_html = tool.help.render(host_url="", static_path="") - cleantext = re.sub(HTML_CLEANER, '', raw_html).replace('\n', '') + cleantext = re.sub(HTML_CLEANER, '', raw_html).replace('\n', ' ') add_doc_kwds['help'] = to_unicode(cleantext) except Exception: # Don't fail to build index just because a help message From 17d67e24c072c8953e482aff41e9b1faed78b4d3 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Mon, 17 Sep 2018 15:31:30 -0400 Subject: [PATCH 12/17] use bleach isntead of custom regex thanks @nsoranzo --- lib/galaxy/tools/search/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/search/__init__.py b/lib/galaxy/tools/search/__init__.py index 19d9fa46e09..7f68703bc06 100644 --- a/lib/galaxy/tools/search/__init__.py +++ b/lib/galaxy/tools/search/__init__.py @@ -6,6 +6,7 @@ import logging import re import tempfile +from bleach import clean from whoosh import analysis from whoosh.analysis import StandardAnalyzer from whoosh.fields import ( @@ -25,7 +26,6 @@ from galaxy.util import ExecutionTimer from galaxy.web.framework.helpers import to_unicode log = logging.getLogger(__name__) -HTML_CLEANER = re.compile("<[^<]+?>") class ToolBoxSearch(object): @@ -107,7 +107,7 @@ class ToolBoxSearch(object): if index_help and tool.help: try: raw_html = tool.help.render(host_url="", static_path="") - cleantext = re.sub(HTML_CLEANER, '', raw_html).replace('\n', ' ') + cleantext = clean(raw_html, tags=[''], strip=True).replace('\n', ' ') add_doc_kwds['help'] = to_unicode(cleantext) except Exception: # Don't fail to build index just because a help message From 54d137edab3fb33beac87c23cd35fb19dd7f44fa Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 13 Sep 2018 15:49:56 +0100 Subject: [PATCH 13/17] Remove deprecated `decode_param_names` parameter It has been deprecated since WebOb 1.1 and ignored since WebOb 1.2 . --- lib/galaxy/web/framework/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/web/framework/base.py b/lib/galaxy/web/framework/base.py index dfc42e95508..b2ef292deb1 100644 --- a/lib/galaxy/web/framework/base.py +++ b/lib/galaxy/web/framework/base.py @@ -369,7 +369,7 @@ class Request(webob.Request): Create a new request wrapping the WSGI environment `environ` """ # self.environ = environ - webob.Request.__init__(self, environ, charset='utf-8', decode_param_names=False) + webob.Request.__init__(self, environ, charset='utf-8') # Properties that are computed and cached on first use @lazy_property From 8ed19a3a09df9b66266f66bc53e504c781ae524d Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 17 Sep 2018 10:04:17 +0100 Subject: [PATCH 14/17] `async` is a reserved keyword in Python 3.7 - Remove unused `async` parameter of `stage_data_async()` method. - Rename `async` parameter of `quick_load()` method. Fix https://github.com/galaxyproject/galaxy/issues/6717 . xref. https://github.com/galaxyproject/galaxy/issues/1715 --- lib/galaxy/tools/toolbox/base.py | 6 +++--- lib/galaxy/tools/verify/interactor.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index f29dd2eba83..a6a2438f940 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -711,7 +711,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): def __watch_directory(self, directory, elems, integrated_elems, load_panel_dict, recursive, force_watch=False): - def quick_load(tool_file, async=True): + def quick_load(tool_file, async_load=True): try: tool = self.load_tool(tool_file) self.__add_tool(tool, load_panel_dict, elems) @@ -719,7 +719,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): key = 'tool_%s' % str(tool.id) integrated_elems[key] = tool - if async: + if async_load: self._load_tool_panel() self._save_integrated_tool_panel() return tool.id @@ -736,7 +736,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): if os.path.isdir(child_path) and recursive: self.__watch_directory(child_path, elems, integrated_elems, load_panel_dict, recursive) elif self._looks_like_a_tool(child_path): - quick_load(child_path, async=False) + quick_load(child_path, async_load=False) tool_loaded = True if (tool_loaded or force_watch) and self._tool_watcher: self._tool_watcher.watch_directory(directory, quick_load) diff --git a/lib/galaxy/tools/verify/interactor.py b/lib/galaxy/tools/verify/interactor.py index 616e5b80622..f25760ffc4f 100644 --- a/lib/galaxy/tools/verify/interactor.py +++ b/lib/galaxy/tools/verify/interactor.py @@ -265,7 +265,7 @@ class GalaxyInteractorApi(object): output_id = output_data return output_id - def stage_data_async(self, test_data, history_id, tool_id, async=True): + def stage_data_async(self, test_data, history_id, tool_id): fname = test_data['fname'] tool_input = { "file_type": test_data['ftype'], From 8982212d396c1ab216c5c9ea0656e5231e05d745 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 17 Sep 2018 11:00:08 +0100 Subject: [PATCH 15/17] Replace deprecated `xml.etree.ElementTree.Element.getiterator()` with `iter()` --- lib/galaxy/util/xml_macros.py | 4 ++-- scripts/loc_files/create_all_fasta_loc.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/util/xml_macros.py b/lib/galaxy/util/xml_macros.py index 3f96a3dc94d..f03b8dac8b5 100644 --- a/lib/galaxy/util/xml_macros.py +++ b/lib/galaxy/util/xml_macros.py @@ -141,7 +141,7 @@ def _expand_macro(element, expand_el, macros, tokens): # require this parent_map data structure but elementtree does not # track parents or recongnize .find('..'). # TODO fix this now that we're not using elementtree - parent_map = dict((c, p) for p in element.getiterator() for c in p) + parent_map = dict((c, p) for p in element.iter() for c in p) _xml_replace(expand_el, expanded_elements, parent_map) @@ -150,7 +150,7 @@ def _expand_yield_statements(macro_def, expand_el): expand_el_children = list(expand_el) macro_def_parent_map = \ - dict((c, p) for macro_def_el in macro_def for p in macro_def_el.getiterator() for c in p) + dict((c, p) for macro_def_el in macro_def for p in macro_def_el.iter() for c in p) for yield_el in yield_els: _xml_replace(yield_el, expand_el_children, macro_def_parent_map) diff --git a/scripts/loc_files/create_all_fasta_loc.py b/scripts/loc_files/create_all_fasta_loc.py index 3feefd17927..7fd4b9c8587 100644 --- a/scripts/loc_files/create_all_fasta_loc.py +++ b/scripts/loc_files/create_all_fasta_loc.py @@ -205,11 +205,11 @@ def __main__(): loc_path = None tree = parse(options.data_table_xml) tables = tree.getroot() - for table in tables.getiterator(): + for table in tables.iter(): name = table.attrib.get('name') if name == options.data_table_name: cols = None - for node in table.getiterator(): + for node in table.iter(): if node.tag == 'columns': cols = node.text elif node.tag == 'file': From f0708488aa08393cfdf3ab3e0e45e37982d050d0 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 17 Sep 2018 12:01:02 +0100 Subject: [PATCH 16/17] pytest: disable test collection of classes starting with `Test` Fix warnings like: ``` /home/travis/build/galaxyproject/galaxy/lib/galaxy/tools/verify/test_data.py:29: PytestWarning: cannot collect test class 'TestDataResolver' because it has a __init__ constructor class TestDataResolver(object): ``` appearing e.g. in https://travis-ci.org/galaxyproject/galaxy/jobs/428114139 . See pytest documentation: https://docs.pytest.org/en/latest/reference.html#confval-python_classes --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index 6479864e5de..f01ebabefdb 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,7 @@ [pytest] addopts = --doctest-continue-on-failure log_level = DEBUG +python_classes = python_paths = lib markers = data_manager: marks test as a data_manager test From 376516610ef6ac28cef7a9558adab1a88d8c0c12 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 18 Sep 2018 11:29:58 -0400 Subject: [PATCH 17/17] Bugfix - missing tests for data source sniffing. Ensure #6334 is fixed I think. --- test/api/test_tools.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/api/test_tools.py b/test/api/test_tools.py index a4710a61f4f..928a5b9a070 100644 --- a/test/api/test_tools.py +++ b/test/api/test_tools.py @@ -118,6 +118,27 @@ class ToolsTestCase(api.ApiTestCase): output_details = self.dataset_populator.get_history_dataset_details(history_id, dataset=output) assert output_details["file_ext"] == "bed" + @skip_without_tool("test_data_source") + def test_data_source_sniff_fastqsanger(self): + with self.dataset_populator.test_history() as history_id: + payload = self.dataset_populator.run_tool_payload( + tool_id="test_data_source", + inputs={ + "URL": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.fastqsanger.gz", + "URL_method": "get", + }, + history_id=history_id, + ) + create_response = self._post("tools", data=payload) + self._assert_status_code_is(create_response, 200) + create_object = create_response.json() + self._assert_has_keys(create_object, "outputs") + assert len(create_object["outputs"]) == 1 + output = create_object["outputs"][0] + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + output_details = self.dataset_populator.get_history_dataset_details(history_id, dataset=output) + assert output_details["file_ext"] == "fastqsanger.gz", output_details + @skip_without_tool("test_data_source") def test_data_sources_block_file_parameters(self): with self.dataset_populator.test_history() as history_id: