From e8b482b44e782a6e5760977e2fa73032fcbf4ac5 Mon Sep 17 00:00:00 2001 From: Juleen Graham Date: Fri, 12 Jun 2020 09:27:24 -0400 Subject: [PATCH 001/208] Show favourited tools independent of version --- client/src/components/Panels/utilities.js | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/client/src/components/Panels/utilities.js b/client/src/components/Panels/utilities.js index 76109648f0c..ea64e6a4c0c 100644 --- a/client/src/components/Panels/utilities.js +++ b/client/src/components/Panels/utilities.js @@ -1,10 +1,12 @@ export function filterToolSections(layout, results) { if (results) { + results = normalize_results(results); const filteredLayout = layout.map((section) => { var toolRes = []; if (section.elems) { section.elems.forEach((el) => { - if (!el.text && results.includes(el.id)) { + if ((!el.text && results.includes(el.id)) || + (el.tool_shed_repository && results.includes(el.tool_shed_repository.name))) { toolRes.push(el); } }); @@ -41,6 +43,7 @@ export function filterTools(layout, results) { // do not consider expression tools, unless requested by the workflow editor layout = layout.filter((section) => section.name !== "Expression Tools"); if (results) { + results = normalize_results(results); var toolsResults = []; if (results.length < 1) { return toolsResults; @@ -50,7 +53,8 @@ export function filterTools(layout, results) { layout.map((section) => { if (section.elems) { section.elems.forEach((el) => { - if (!el.text && results.includes(el.id)) { + if ((!el.text && results.includes(el.id)) || + (el.tool_shed_repository && results.includes(el.tool_shed_repository.name))) { toolsResults.push(el); } }); @@ -63,3 +67,13 @@ export function filterTools(layout, results) { return layout; } } + +function normalize_results(results) { + var norm_results = []; + results.forEach((result) => { + if (result.includes("/repos/")) { + norm_results.push(result.split('/')[3]); + } + }); + return norm_results; +} \ No newline at end of file From 1e960f5f6037d68f1960250351cc21e9270b9386 Mon Sep 17 00:00:00 2001 From: Juleen Graham Date: Mon, 21 Sep 2020 10:58:57 -0400 Subject: [PATCH 002/208] fix unrendered results --- client/src/components/Panels/utilities.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/client/src/components/Panels/utilities.js b/client/src/components/Panels/utilities.js index ea64e6a4c0c..8bbd390555f 100644 --- a/client/src/components/Panels/utilities.js +++ b/client/src/components/Panels/utilities.js @@ -74,6 +74,9 @@ function normalize_results(results) { if (result.includes("/repos/")) { norm_results.push(result.split('/')[3]); } + else { + norm_results.push(result); + } }); return norm_results; } \ No newline at end of file From b89cdf33913d99fc03ba1e02912ea4fc49a77eef Mon Sep 17 00:00:00 2001 From: Juleen Graham Date: Tue, 22 Sep 2020 00:09:08 -0400 Subject: [PATCH 003/208] prettier --- client/src/components/Panels/utilities.js | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/client/src/components/Panels/utilities.js b/client/src/components/Panels/utilities.js index 8bbd390555f..75e5996f74c 100644 --- a/client/src/components/Panels/utilities.js +++ b/client/src/components/Panels/utilities.js @@ -5,8 +5,10 @@ export function filterToolSections(layout, results) { var toolRes = []; if (section.elems) { section.elems.forEach((el) => { - if ((!el.text && results.includes(el.id)) || - (el.tool_shed_repository && results.includes(el.tool_shed_repository.name))) { + if ( + (!el.text && results.includes(el.id)) || + (el.tool_shed_repository && results.includes(el.tool_shed_repository.name)) + ) { toolRes.push(el); } }); @@ -53,8 +55,10 @@ export function filterTools(layout, results) { layout.map((section) => { if (section.elems) { section.elems.forEach((el) => { - if ((!el.text && results.includes(el.id)) || - (el.tool_shed_repository && results.includes(el.tool_shed_repository.name))) { + if ( + (!el.text && results.includes(el.id)) || + (el.tool_shed_repository && results.includes(el.tool_shed_repository.name)) + ) { toolsResults.push(el); } }); @@ -72,11 +76,10 @@ function normalize_results(results) { var norm_results = []; results.forEach((result) => { if (result.includes("/repos/")) { - norm_results.push(result.split('/')[3]); - } - else { + norm_results.push(result.split("/")[3]); + } else { norm_results.push(result); } }); return norm_results; -} \ No newline at end of file +} From 2165d71d328181709ea64be1f93f76be61203974 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Thu, 22 Oct 2020 15:12:09 +0200 Subject: [PATCH 004/208] proper error msg --- lib/galaxy/webapps/galaxy/api/authenticate.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/authenticate.py b/lib/galaxy/webapps/galaxy/api/authenticate.py index 570ad83cc52..c3ea5cafca9 100644 --- a/lib/galaxy/webapps/galaxy/api/authenticate.py +++ b/lib/galaxy/webapps/galaxy/api/authenticate.py @@ -92,7 +92,14 @@ class AuthenticationController(BaseAPIController): :raises: HTTPBadRequest """ - split = encoded_str.strip().split(' ') + try: + split = encoded_str.strip().split(' ') + except AttributeError: + if not encoded_str: + raise exceptions.MessageException( + 'Authentication is missing') + raise exceptions.MessageException( + 'Failed to authenticate') # If split is only one element, try to decode the email and password # directly. From c54015466bb1851762a9e90cebc580ca1e9f9863 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Thu, 22 Oct 2020 15:22:48 +0200 Subject: [PATCH 005/208] formating --- lib/galaxy/webapps/galaxy/api/authenticate.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/authenticate.py b/lib/galaxy/webapps/galaxy/api/authenticate.py index c3ea5cafca9..d8d03eaff0d 100644 --- a/lib/galaxy/webapps/galaxy/api/authenticate.py +++ b/lib/galaxy/webapps/galaxy/api/authenticate.py @@ -98,8 +98,9 @@ class AuthenticationController(BaseAPIController): if not encoded_str: raise exceptions.MessageException( 'Authentication is missing') - raise exceptions.MessageException( - 'Failed to authenticate') + else: + raise exceptions.MessageException( + 'Failed to authenticate') # If split is only one element, try to decode the email and password # directly. From ef155452cf55255afec6097e4a6c3c5ddef40a95 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Thu, 22 Oct 2020 16:23:14 +0200 Subject: [PATCH 006/208] Update lib/galaxy/webapps/galaxy/api/authenticate.py Co-authored-by: Dannon --- lib/galaxy/webapps/galaxy/api/authenticate.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/authenticate.py b/lib/galaxy/webapps/galaxy/api/authenticate.py index d8d03eaff0d..e3a6bb103bf 100644 --- a/lib/galaxy/webapps/galaxy/api/authenticate.py +++ b/lib/galaxy/webapps/galaxy/api/authenticate.py @@ -96,11 +96,9 @@ class AuthenticationController(BaseAPIController): split = encoded_str.strip().split(' ') except AttributeError: if not encoded_str: - raise exceptions.MessageException( - 'Authentication is missing') + raise exceptions.MessageException('Authentication is missing') else: - raise exceptions.MessageException( - 'Failed to authenticate') + raise exceptions.MessageException('Failed to authenticate') # If split is only one element, try to decode the email and password # directly. From 754da6bcc60476d1758a56f500853a8da4e29442 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Fri, 23 Oct 2020 13:03:44 +0200 Subject: [PATCH 007/208] Update lib/galaxy/webapps/galaxy/api/authenticate.py Co-authored-by: Dannon --- lib/galaxy/webapps/galaxy/api/authenticate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/authenticate.py b/lib/galaxy/webapps/galaxy/api/authenticate.py index e3a6bb103bf..475347957dd 100644 --- a/lib/galaxy/webapps/galaxy/api/authenticate.py +++ b/lib/galaxy/webapps/galaxy/api/authenticate.py @@ -96,7 +96,7 @@ class AuthenticationController(BaseAPIController): split = encoded_str.strip().split(' ') except AttributeError: if not encoded_str: - raise exceptions.MessageException('Authentication is missing') + raise exceptions.RequestParameterInvalidException('Authentication is missing') else: raise exceptions.MessageException('Failed to authenticate') From 19a783ca04cf4e6a222bcfd89705741cdf24af1e Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Fri, 23 Oct 2020 13:16:25 +0200 Subject: [PATCH 008/208] remove if condition --- lib/galaxy/webapps/galaxy/api/authenticate.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/authenticate.py b/lib/galaxy/webapps/galaxy/api/authenticate.py index 475347957dd..ac29256dce0 100644 --- a/lib/galaxy/webapps/galaxy/api/authenticate.py +++ b/lib/galaxy/webapps/galaxy/api/authenticate.py @@ -95,10 +95,8 @@ class AuthenticationController(BaseAPIController): try: split = encoded_str.strip().split(' ') except AttributeError: - if not encoded_str: - raise exceptions.RequestParameterInvalidException('Authentication is missing') - else: - raise exceptions.MessageException('Failed to authenticate') + raise exceptions.RequestParameterInvalidException('Authentication is missing') + # If split is only one element, try to decode the email and password # directly. From eccf4e3f35e10ab5e2c95bf13b5adfb5ff6075b4 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Fri, 23 Oct 2020 08:10:03 -0400 Subject: [PATCH 009/208] Minor lint fix --- lib/galaxy/webapps/galaxy/api/authenticate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/authenticate.py b/lib/galaxy/webapps/galaxy/api/authenticate.py index ac29256dce0..914bbbedf69 100644 --- a/lib/galaxy/webapps/galaxy/api/authenticate.py +++ b/lib/galaxy/webapps/galaxy/api/authenticate.py @@ -97,7 +97,6 @@ class AuthenticationController(BaseAPIController): except AttributeError: raise exceptions.RequestParameterInvalidException('Authentication is missing') - # If split is only one element, try to decode the email and password # directly. if len(split) == 1: From 094dd1adb1fe9ecf5e5cb222e9ea79b162822375 Mon Sep 17 00:00:00 2001 From: guerler Date: Thu, 17 Dec 2020 18:45:21 +0100 Subject: [PATCH 010/208] Allow toggling favorites button in tool panel --- client/src/components/Panels/Buttons/FavoritesButton.vue | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/client/src/components/Panels/Buttons/FavoritesButton.vue b/client/src/components/Panels/Buttons/FavoritesButton.vue index 82fb53f2251..59eb6585207 100644 --- a/client/src/components/Panels/Buttons/FavoritesButton.vue +++ b/client/src/components/Panels/Buttons/FavoritesButton.vue @@ -25,6 +25,7 @@ export default { data() { return { tooltipText: "Show favorites", + toggle: false, }; }, directives: { @@ -32,7 +33,12 @@ export default { }, methods: { onFavorites() { - this.$emit("onFavorites", "#favorites"); + this.toggle = !this.toggle; + if (this.toggle) { + this.$emit("onFavorites", "#favorites"); + } else { + this.$emit("onFavorites", null); + } }, }, }; From 157027615f0e9c0ceeb8f1c065a30120fe3f3afd Mon Sep 17 00:00:00 2001 From: guerler Date: Thu, 17 Dec 2020 18:48:29 +0100 Subject: [PATCH 011/208] Fill star icon if favorites --- client/src/components/Panels/Buttons/FavoritesButton.vue | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/client/src/components/Panels/Buttons/FavoritesButton.vue b/client/src/components/Panels/Buttons/FavoritesButton.vue index 59eb6585207..6a3101dda88 100644 --- a/client/src/components/Panels/Buttons/FavoritesButton.vue +++ b/client/src/components/Panels/Buttons/FavoritesButton.vue @@ -8,7 +8,8 @@ role="button" aria-label="Show favorite tools" > - + + @@ -17,7 +18,10 @@ import { VBTooltip } from "bootstrap-vue"; import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome"; import { library } from "@fortawesome/fontawesome-svg-core"; import { faStar } from "@fortawesome/free-regular-svg-icons"; +import { faStar as faStarSolid } from "@fortawesome/free-solid-svg-icons"; + library.add(faStar); +library.add(faStarSolid); export default { name: "FavoritesButton", From a6a1764b90139bd08eaa1c9493334f17b8d80ae1 Mon Sep 17 00:00:00 2001 From: guerler Date: Thu, 17 Dec 2020 19:04:43 +0100 Subject: [PATCH 012/208] Detect search term changes in tool panel --- .../components/Panels/Buttons/FavoritesButton.vue | 13 ++++++++++++- client/src/components/Panels/ToolBox.vue | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/client/src/components/Panels/Buttons/FavoritesButton.vue b/client/src/components/Panels/Buttons/FavoritesButton.vue index 6a3101dda88..6200b56bca0 100644 --- a/client/src/components/Panels/Buttons/FavoritesButton.vue +++ b/client/src/components/Panels/Buttons/FavoritesButton.vue @@ -25,9 +25,15 @@ library.add(faStarSolid); export default { name: "FavoritesButton", + props: { + query: { + type: String, + }, + }, components: { FontAwesomeIcon }, data() { return { + searchKey: "#favorites", tooltipText: "Show favorites", toggle: false, }; @@ -35,11 +41,16 @@ export default { directives: { "v-b-tooltip": VBTooltip, }, + watch: { + query() { + this.toggle = this.query == this.searchKey; + }, + }, methods: { onFavorites() { this.toggle = !this.toggle; if (this.toggle) { - this.$emit("onFavorites", "#favorites"); + this.$emit("onFavorites", this.searchKey); } else { this.$emit("onFavorites", null); } diff --git a/client/src/components/Panels/ToolBox.vue b/client/src/components/Panels/ToolBox.vue index 472203b731e..14a66c8bf65 100644 --- a/client/src/components/Panels/ToolBox.vue +++ b/client/src/components/Panels/ToolBox.vue @@ -3,7 +3,7 @@
- +
Tools
From 7b3ff4398c4c853d1c5e89e3833149bef33c8dd6 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 18 Dec 2020 09:40:34 +0100 Subject: [PATCH 013/208] Align favorite button tooltip to toggle state --- .../components/Panels/Buttons/FavoritesButton.vue | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/client/src/components/Panels/Buttons/FavoritesButton.vue b/client/src/components/Panels/Buttons/FavoritesButton.vue index 6200b56bca0..a0b295b7ca5 100644 --- a/client/src/components/Panels/Buttons/FavoritesButton.vue +++ b/client/src/components/Panels/Buttons/FavoritesButton.vue @@ -34,7 +34,8 @@ export default { data() { return { searchKey: "#favorites", - tooltipText: "Show favorites", + tooltipToggle: "Show favorites", + tooltipUntoggle: "Clear", toggle: false, }; }, @@ -46,6 +47,15 @@ export default { this.toggle = this.query == this.searchKey; }, }, + computed: { + tooltipText() { + if (this.toggle) { + return this.tooltipUntoggle; + } else { + return this.tooltipToggle; + } + }, + }, methods: { onFavorites() { this.toggle = !this.toggle; From 1278994762904c87c5b00e1de349d7a927d70b9f Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Mon, 21 Dec 2020 15:30:44 -0500 Subject: [PATCH 014/208] If input file is empty, and you are trying to access anything but column 1 of the input file, you will get a validation error. This change fixes that so empty input files can be processed (generate empty output files). --- lib/galaxy/tools/parameters/basic.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 36618d478a5..2f6ad71b011 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -936,6 +936,11 @@ class SelectToolParameter(ToolParameter): if is_runtime_value(value): return None if value not in legal_values and require_legal_value: + if isinstance(self, ColumnListParameter): + hda = other_values.get(self.data_ref) + if hda is not None and hda.dataset.get_size() == 0: + # If dataset is empty, should not invalidate columns not equal to 1 + return value raise ParameterValueError("an invalid option ({!r}) was selected (valid options: {})".format(value, ",".join(legal_values)), self.name, value, is_dynamic=self.is_dynamic) return value From b052c7e39dfe8718699a1209c2a1aaca6959e694 Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Tue, 22 Dec 2020 11:48:42 -0500 Subject: [PATCH 015/208] Moved the logic to get_column_list() method --- lib/galaxy/tools/parameters/basic.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 2f6ad71b011..a316daf46ac 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -936,11 +936,6 @@ class SelectToolParameter(ToolParameter): if is_runtime_value(value): return None if value not in legal_values and require_legal_value: - if isinstance(self, ColumnListParameter): - hda = other_values.get(self.data_ref) - if hda is not None and hda.dataset.get_size() == 0: - # If dataset is empty, should not invalidate columns not equal to 1 - return value raise ParameterValueError("an invalid option ({!r}) was selected (valid options: {})".format(value, ",".join(legal_values)), self.name, value, is_dynamic=self.is_dynamic) return value @@ -1303,6 +1298,11 @@ class ColumnListParameter(SelectToolParameter): column_list = this_column_list else: column_list = [c for c in column_list if c in this_column_list] + + if dataset.dataset.get_size() == 0: + value = other_values.get(self.name) + if value is not None: + column_list.append(value) return column_list def get_options(self, trans, other_values): From 3fa17f706818e1aecec09fc6fa829c88d45c8578 Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Tue, 22 Dec 2020 15:43:13 -0500 Subject: [PATCH 016/208] Stop making new sections for every toolshed tool. This change will place a newly installed toolshed tool, if the target section exists, in that previously defined section in the shed_tool_conf file. Previously the section would simply be placed at the very end, whether or not that section already existed in the XML. --- .../galaxy_install/tools/tool_panel_manager.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py b/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py index 5e438bb6b2f..8e2d7e16716 100644 --- a/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -56,7 +56,21 @@ class ToolPanelManager: config_elems.append(elem) # Add the new elements to the in-memory list of config_elems. for elem_entry in elem_list: - config_elems.append(elem_entry) + section_found = False + if elem_entry.tag == 'section': + # See if there is already a section with a matching ID + for existing_elem in config_elems: + if existing_elem.tag == 'section' and existing_elem.attrib.get('id', None) == elem_entry.attrib.get('id', None): + section_found = True + for child in elem_entry: + existing_elem.append(child) + break + if section_found: + break + if not section_found: + config_elems.append(elem_entry) + else: + config_elems.append(elem_entry) # Persist the altered shed_tool_config file. self.config_elems_to_xml_file(config_elems, shed_tool_conf, tool_path, tool_cache_data_dir) self.app.wait_for_toolbox_reload(old_toolbox) From 67fd5860fd270d60720f2f332c0033d3805ccb20 Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Wed, 23 Dec 2020 10:43:07 -0500 Subject: [PATCH 017/208] Moved the logic (back) to get_legal_values() method. --- lib/galaxy/tools/parameters/basic.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index a316daf46ac..eeccfd2c920 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1298,11 +1298,6 @@ class ColumnListParameter(SelectToolParameter): column_list = this_column_list else: column_list = [c for c in column_list if c in this_column_list] - - if dataset.dataset.get_size() == 0: - value = other_values.get(self.name) - if value is not None: - column_list.append(value) return column_list def get_options(self, trans, other_values): @@ -1341,7 +1336,15 @@ class ColumnListParameter(SelectToolParameter): def get_legal_values(self, trans, other_values): if self.data_ref not in other_values: raise ValueError("Value for associated data reference not found (data_ref).") - return set(self.get_column_list(trans, other_values)) + legal_values = self.get_column_list(trans, other_values) + + hda = other_values.get(self.data_ref) + if hda is not None and hda.dataset.get_size() == 0: + value = other_values.get(self.name) + if value is not None: + legal_values.append(value) + + return set(legal_values) def get_dependencies(self): return [self.data_ref] From 3b15a2148e7c9ebcff0aaef57772115aea4ce170 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 1 Jan 2021 19:45:53 +0100 Subject: [PATCH 018/208] document how parameters in sections are referenced --- lib/galaxy/tool_util/xsd/galaxy.xsd | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 9b0e27e5245..cd39788468b 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2383,8 +2383,8 @@ The XML configuration is relatively trivial for sections: ```xml -
- +
+
``` @@ -2393,9 +2393,25 @@ In your command template, you'll need to include the section name to access the variable: ``` ---color $adv.plot_color +$section_name.parameter_name ``` +In output filters sections are represented as dictionary with the same name as the section: + +``` +section_name['parameter_name'] +``` + +In order to reference parameters in sections from tags in the `` section, e.g. in the `format_source` attribute of `` tags, the syntax is: + +``` +section_name|parameter_name +``` + +Until profile 21.01 `parameter_name` was sufficient (https://github.com/galaxyproject/galaxy/pull/9493/files). + +Note that references to other parameters in the `` section are only possible if the reference is in the same section or its parents, therefor only `parameter_name` is used. + Further examples can be found in the [test case](https://github.com/galaxyproject/galaxy/blob/master/test/functional/tools/section.xml) from [pull request #35](https://github.com/galaxyproject/galaxy/pull/35). ]]> @@ -4386,6 +4402,7 @@ parameters appear as booleans, not the value of their ``truevalue`` and appear as ``$options.selection_mode`` in Cheetah. Similarly ``options["vcf_output"]`` would appear as ``$options.vcf_output`` having the values ``'--vcf'`` when true and ``''`` when false in Cheetah. +Note that also parameters in sections are accessed via a dictionary. ### Example From ce31be70227e73404a30b8a750a31b56e30b493d Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 4 Jan 2021 13:08:57 +0100 Subject: [PATCH 019/208] improve docs for format_source xref to sections --- lib/galaxy/tool_util/xsd/galaxy.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index cd39788468b..33c260ee481 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2405,7 +2405,7 @@ In output filters sections are represented as dictionary with the same name as t In order to reference parameters in sections from tags in the `` section, e.g. in the `format_source` attribute of `` tags, the syntax is: ``` -section_name|parameter_name + ``` Until profile 21.01 `parameter_name` was sufficient (https://github.com/galaxyproject/galaxy/pull/9493/files). From dea1f92fbcae390b6e1e1e3c0db91b9d8d432b29 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 4 Jan 2021 13:09:21 +0100 Subject: [PATCH 020/208] improve docs for data_ref and sections --- lib/galaxy/tool_util/xsd/galaxy.xsd | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 33c260ee481..c5bc7c47d1c 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2410,7 +2410,28 @@ In order to reference parameters in sections from tags in the `` sectio Until profile 21.01 `parameter_name` was sufficient (https://github.com/galaxyproject/galaxy/pull/9493/files). -Note that references to other parameters in the `` section are only possible if the reference is in the same section or its parents, therefor only `parameter_name` is used. +Note that references to other parameters in the `` section are only possible if the reference is in the same section or its parents (and is defined earlier), therefore only `parameter_name` is used. + +``` + + +
+ + + +
+``` + +In the above example `bar` and `qux` will refer to the first foo outside of the section and `baz` to the `foo` inside the section. This illustrates why non-unique parameter names are strongly discouraged. + +The following will not work: + +``` +
+ +
+ +``` Further examples can be found in the [test case](https://github.com/galaxyproject/galaxy/blob/master/test/functional/tools/section.xml) from [pull request #35](https://github.com/galaxyproject/galaxy/pull/35). ]]> From 865736aeca7855df6bc6b4700216cd2f19b5d144 Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Tue, 5 Jan 2021 08:24:59 -0500 Subject: [PATCH 021/208] Handled case where we have a dataset collection --- lib/galaxy/tools/parameters/basic.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index eeccfd2c920..a16292fab58 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1339,10 +1339,22 @@ class ColumnListParameter(SelectToolParameter): legal_values = self.get_column_list(trans, other_values) hda = other_values.get(self.data_ref) - if hda is not None and hda.dataset.get_size() == 0: - value = other_values.get(self.name) - if value is not None: - legal_values.append(value) + if isinstance(hda, list): + # hda is a list of HistoryDatasetCollectionAssociation + for idx1 in range(len(hda)): + # dataset_instances is a list of HistoryDatasetAssociation + dataset_instances = hda[idx1].dataset_instances + for idx2 in range(len(dataset_instances)): + if dataset_instances[idx2] is not None and dataset_instances[idx2].get_size() == 0: + value = other_values.get(self.name) + if value is not None: + legal_values.append(value) + else: + # hda is a HistoryDatasetAssociation + if hda.dataset.get_size() == 0: + value = other_values.get(self.name) + if value is not None: + legal_values.append(value) return set(legal_values) From 816dd8be163f9b5880dac9605baee0250895d972 Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Tue, 5 Jan 2021 09:52:15 -0500 Subject: [PATCH 022/208] Remove a break that shouldn't be there, add comments. --- .../tool_shed/galaxy_install/tools/tool_panel_manager.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py b/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py index 8e2d7e16716..f451f6e7cf1 100644 --- a/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -58,13 +58,14 @@ class ToolPanelManager: for elem_entry in elem_list: section_found = False if elem_entry.tag == 'section': - # See if there is already a section with a matching ID + # See if there is already a section with a matching ID. for existing_elem in config_elems: if existing_elem.tag == 'section' and existing_elem.attrib.get('id', None) == elem_entry.attrib.get('id', None): section_found = True + # We've found a section, append everything from this
tag to the one in memory. for child in elem_entry: existing_elem.append(child) - break + # And jump to the next incoming section. if section_found: break if not section_found: From aaf44910b8e05e7870e108ba43c229cdb1b900ff Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Tue, 5 Jan 2021 11:22:44 -0500 Subject: [PATCH 023/208] Revised code based on review. --- lib/galaxy/tools/parameters/basic.py | 39 ++++++++++++++++------------ 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index a16292fab58..5e3a95fff32 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1338,26 +1338,31 @@ class ColumnListParameter(SelectToolParameter): raise ValueError("Value for associated data reference not found (data_ref).") legal_values = self.get_column_list(trans, other_values) - hda = other_values.get(self.data_ref) - if isinstance(hda, list): - # hda is a list of HistoryDatasetCollectionAssociation - for idx1 in range(len(hda)): - # dataset_instances is a list of HistoryDatasetAssociation - dataset_instances = hda[idx1].dataset_instances - for idx2 in range(len(dataset_instances)): - if dataset_instances[idx2] is not None and dataset_instances[idx2].get_size() == 0: - value = other_values.get(self.name) - if value is not None: - legal_values.append(value) - else: - # hda is a HistoryDatasetAssociation - if hda.dataset.get_size() == 0: - value = other_values.get(self.name) - if value is not None: - legal_values.append(value) + value = other_values.get(self.name) + if value not in legal_values: + legal_values.extend(self.get_empty_file_columns(trans, other_values)) return set(legal_values) + def get_empty_file_columns(self, trans, other_values): + # Get the value of the associated data reference (a dataset) + dataset = other_values.get(self.data_ref) + + # Check if a dataset is selected + if not dataset: + return [] + + column_list = [] + for dataset in util.listify(dataset): + # Use representative dataset if a dataset collection is parsed + if isinstance(dataset, trans.app.model.HistoryDatasetCollectionAssociation): + dataset = dataset.to_hda_representative() + if dataset.get_size() == 0: + value = other_values.get(self.name) + if value is not None: + column_list.append(value) + return column_list + def get_dependencies(self): return [self.data_ref] From b7382469e9214ba84eb0ee916f532f121825e5c0 Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Tue, 5 Jan 2021 14:01:53 -0500 Subject: [PATCH 024/208] More code revision based on review comments. --- lib/galaxy/tools/parameters/basic.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 5e3a95fff32..70c304fcbd5 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1339,29 +1339,19 @@ class ColumnListParameter(SelectToolParameter): legal_values = self.get_column_list(trans, other_values) value = other_values.get(self.name) - if value not in legal_values: - legal_values.extend(self.get_empty_file_columns(trans, other_values)) + if value not in legal_values and self.is_file_empty(trans, other_values): + legal_values.extend(value) return set(legal_values) - def get_empty_file_columns(self, trans, other_values): - # Get the value of the associated data reference (a dataset) - dataset = other_values.get(self.data_ref) - - # Check if a dataset is selected - if not dataset: - return [] - - column_list = [] - for dataset in util.listify(dataset): + def is_file_empty(self, trans, other_values): + for dataset in util.listify(other_values.get(self.data_ref)): # Use representative dataset if a dataset collection is parsed if isinstance(dataset, trans.app.model.HistoryDatasetCollectionAssociation): dataset = dataset.to_hda_representative() - if dataset.get_size() == 0: - value = other_values.get(self.name) - if value is not None: - column_list.append(value) - return column_list + if not dataset.has_data(): + return True + return False def get_dependencies(self): return [self.data_ref] From f66444aa17c9c17efa59204aee14152136f0260b Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Tue, 5 Jan 2021 14:40:38 -0500 Subject: [PATCH 025/208] Fixed broken unit test --- lib/galaxy/tools/parameters/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 70c304fcbd5..5a4101e2f0a 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1339,7 +1339,7 @@ class ColumnListParameter(SelectToolParameter): legal_values = self.get_column_list(trans, other_values) value = other_values.get(self.name) - if value not in legal_values and self.is_file_empty(trans, other_values): + if value is not None and value not in legal_values and self.is_file_empty(trans, other_values): legal_values.extend(value) return set(legal_values) From e03165ad414435ada4dd81fcc6481e5e960f8df9 Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Fri, 8 Jan 2021 09:25:17 -0500 Subject: [PATCH 026/208] Simplify logic slightly, add more comments. --- .../galaxy_install/tools/tool_panel_manager.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py b/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py index f451f6e7cf1..e76aa6366f2 100644 --- a/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -56,20 +56,24 @@ class ToolPanelManager: config_elems.append(elem) # Add the new elements to the in-memory list of config_elems. for elem_entry in elem_list: + # Initialize section_found to False. section_found = False if elem_entry.tag == 'section': - # See if there is already a section with a matching ID. + # Loop through section entries in the in-memory tool panel. for existing_elem in config_elems: + # Compare the section ID for each one to the section ID for the tool being installed. if existing_elem.tag == 'section' and existing_elem.attrib.get('id', None) == elem_entry.attrib.get('id', None): + # We've found a section, set section_found to True so the subsequent if statement won't append it to the end. section_found = True - # We've found a section, append everything from this
tag to the one in memory. for child in elem_entry: existing_elem.append(child) - # And jump to the next incoming section. - if section_found: - break + # Break out of the config_elems loop back to the elem_list loop so that section_found is reset to False + # and we can process the next section, if any. + break + # If an existing section with a matching ID was not found, create that section and its tools. if not section_found: config_elems.append(elem_entry) + # This is not a section, but a tool or label. No need to search for matching sections, just add it. else: config_elems.append(elem_entry) # Persist the altered shed_tool_config file. From 7ca7f74af80a4136e6df9e9ccf253948b19e2567 Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Fri, 8 Jan 2021 09:29:53 -0500 Subject: [PATCH 027/208] Simplify logic even further. --- .../galaxy_install/tools/tool_panel_manager.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py b/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py index e76aa6366f2..94fc336d23a 100644 --- a/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -56,23 +56,18 @@ class ToolPanelManager: config_elems.append(elem) # Add the new elements to the in-memory list of config_elems. for elem_entry in elem_list: - # Initialize section_found to False. - section_found = False if elem_entry.tag == 'section': # Loop through section entries in the in-memory tool panel. for existing_elem in config_elems: # Compare the section ID for each one to the section ID for the tool being installed. if existing_elem.tag == 'section' and existing_elem.attrib.get('id', None) == elem_entry.attrib.get('id', None): # We've found a section, set section_found to True so the subsequent if statement won't append it to the end. - section_found = True for child in elem_entry: existing_elem.append(child) - # Break out of the config_elems loop back to the elem_list loop so that section_found is reset to False - # and we can process the next section, if any. + # Break out of the config_elems loop back to the elem_list loop break - # If an existing section with a matching ID was not found, create that section and its tools. - if not section_found: - config_elems.append(elem_entry) + # If we reach this point, no section was found. Create one with contents. + config_elems.append(elem_entry) # This is not a section, but a tool or label. No need to search for matching sections, just add it. else: config_elems.append(elem_entry) From 009375163b4caaa39ca669fafd5580ebd805cb97 Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Fri, 8 Jan 2021 09:32:36 -0500 Subject: [PATCH 028/208] Update comment. --- .../tool_shed/galaxy_install/tools/tool_panel_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py b/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py index 94fc336d23a..ff604fd6d19 100644 --- a/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -61,10 +61,10 @@ class ToolPanelManager: for existing_elem in config_elems: # Compare the section ID for each one to the section ID for the tool being installed. if existing_elem.tag == 'section' and existing_elem.attrib.get('id', None) == elem_entry.attrib.get('id', None): - # We've found a section, set section_found to True so the subsequent if statement won't append it to the end. + # We've found a section, add the incoming tools to it. for child in elem_entry: existing_elem.append(child) - # Break out of the config_elems loop back to the elem_list loop + # Break out of the config_elems loop back to the elem_list loop. break # If we reach this point, no section was found. Create one with contents. config_elems.append(elem_entry) From d7ba38915f232d78055c7143f38b5c405cb82da9 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 2 Jan 2021 23:06:44 +0100 Subject: [PATCH 029/208] Add the 3D Visualisation from Nora add nora change nora plugin (rebased to drop artifacts) --- client/gulpfile.js | 1 + .../visualizations/nora/__static/logo.png | Bin 0 -> 5502 bytes .../nora/__static/plugin_build_hash.txt | 1 + .../visualizations/nora/config/nora.xml | 27 ++++++++++++++++++ .../plugins/visualizations/nora/package.json | 18 ++++++++++++ 5 files changed, 47 insertions(+) create mode 100644 config/plugins/visualizations/nora/__static/logo.png create mode 100644 config/plugins/visualizations/nora/__static/plugin_build_hash.txt create mode 100644 config/plugins/visualizations/nora/config/nora.xml create mode 100644 config/plugins/visualizations/nora/package.json diff --git a/client/gulpfile.js b/client/gulpfile.js index 015f572fda8..727ea665282 100644 --- a/client/gulpfile.js +++ b/client/gulpfile.js @@ -22,6 +22,7 @@ const PLUGIN_BUILD_IDS = [ "openlayers", "openseadragon", "pv", + "nora", ]; const paths = { diff --git a/config/plugins/visualizations/nora/__static/logo.png b/config/plugins/visualizations/nora/__static/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..987efb0a6d24be06ce96ba05ba5d4e8fa707363f GIT binary patch literal 5502 zcmb_g^;Z;J6J9!&5?8t#K|&;?L0~CCI>nX7pu4+!>6Ff8L2`j5q(mg7J49NfmtMYo z|A%kRx$(=LGxy$^XJ($5H(Fp)Vme{~06?mys-%lqyD%ez5F0Ze?Ukiq79wX=6E^^W ztpC4(mCi>-4*)z7Qd3gU_s%=acW~ANw+xvX&3+ik;1v9x+iAJr-AgZ)HGl$ODO&V& z@{1Dr{IQ|^9^P3k1W(;Chz;jYe~O>V#+uAJ{S$=sje~+Rm4pHX5gNpgKM56{XX$kM z%4u?oJPM1UXvvtpNbg8Tbdw9Xc}!m#-_5MF-sVj)#1;d9ahDX8jF1x};KmV`;F8(o^9iTWrH!yFn3fXl|xzQJ5CKoT0bw%uB^P?L`pMhb5JU)s60*oQCeEs z+1VK%Pa<15-UgKo4ZEB2B3IVof(}E3vNXA*-->2y-l7hr+;@{lZF3=d*dY+q--S48 zK>-1Q?Gx|#wjHVCbTVZfc6N5j{6r45=`Qc`=4Own^4A}lb5ds6*x3txwzIcJGUJrk zROrs}vefOYt(9tKo(s^$7VkTGiPrcvvzN=W%JFjX^QrZ2Jr~&!p&>DA`BYU~Yh5Y| zkT*7-Uu^!E$#3#)!#V2%RG}W4IBGj(|H8JBc5vR3(`k9n@sMF+KLIO4y{7qF{V zDDTmbR4H+>w?SH3%d$!@};o7I^!4T6TE|B4q!9j)r$h z)pc&;M+vo47XPfUpbqGb*GB=-%BjcfoR%AK>MHFn*(JFbznes3i*t@pKS$NJDYXXi zpaY9il~cPU!NuYJl~o@V6&3aM^=)i8_1H{2@;P{u6OG8shRo^&zbD+HCpA7gT ziUHwYy}iA_*BV&&p{}~ys%6oVy$q|M9DH(eV%czYx;2swl@7RZ_xAQSHKo)0 ztVO|JY9&Oo?lHCatNYSEVMpvWRpUTLb}0a0-sJ)>Y92u%r8$IT-v8J=GNixWO4$!8 zXj`oLC52+}EkRVlI@;Cp>8P0)cKuasz-77@DAa0Kz*0%gG3JgOT>r`Qd9KeBa4s{L zq4x>^p!Lo%gn{5`U2|fhy;$y!L)3*4eJpm+Lqqj%(;B{!RfUy|06Sj-52hvm>#fWu z#CU0N=VXrSSu-b5ad8ohDhiFL;>zA$1^71Z>(4UD8B{fgo_=xw`A1ZLf6dF98i+0< zrk&@I4P%g-r6dC;Bvp)(Ao83t4L*@Ix-Guy{R%!OvcykHp#iR$F@;%Gv^##``QY8Z zyYSBlzZoAOZ|YxK!;R9xy$jycD**?2$?<)4JFr-Tp=6V6@$m4nv$KUrzm*u)$lago z*)El0sqbOp)TvvBfh>x;C!&@5xD5cX09;mGzjU0P$rvAD2*^5`*_FJFWTl5A7s#v( zEiA|uSfmNa7%Jg#yG|3hJTD{6@L;t^C6zA!egD;j<{o+9pRq+JXFU}t3GvC zD7y`1M}iR_I(d+R%x>YUjajSXMX>{+z}t5Btsm)mc?`C>5d_)pDk|oIHm|7;GLRKZ zk_3N^7X6|;8EUV_zy8G&(cRhNC~R|gdvkD2-~GvPFpIB<(#1zDBezv4TwQ_z@baVA zbe^BOy1M7ls*HpLp#%AFSHLZWI>AD)jS!7F(!J^Hu#UKAmyF(=Naf~PTad9Vbo@+c z)5JI{C!JvnW>iF}(~0fb%47F|q?9vh$gIgerFG2v&oaWTudiN2WKe;f5jy5w zKx4zGilO)_9)Lq>_~HCk*C%?uiZ)XtEz>$Z(|YRcvxAFd{K9$K?z}3me`8r$+4-GI z*up1SpHq&_QN>Ud$$#R5qNbUQlga#_L~-ohJ9#QEmCm)G8LKYH5}4K|fYkSJOB>rhMVXcsWS~#c%gj%!tC3 zwePnCl7Y506AGaD6pT#OgqeiiKkr`hsW&QPuaZBx;MlXCh0<^a`CRiS(LdT89v^pd zaCrZ%mq6dhNMwmc+P-u%khWfeZzC7lya_+^S*D%(4FG`f67|KjI1<9q_GvQ9Ms!ay)YnTN7Na#R%JY!6xM&LHMx`ez?#RW)sx1;6<=o8_(Adfl} zs;3^^%)RsciJrK!fC)l^831XS*R+l6zjd@Fd72(OzVk|@p?7L_Hd0BFa9fhe>Ps8$ ziMr2;-kBT5QQL4?*9ok-!1qUuWg?TC06_kI-;m(TwTF-PhjotDT$1yTWRs*^Fc5-` zjqNYIvbTRq0a|CLIF}U4&ByKK9nt0;@uFOCN9+|$@qXu-N!Sl{kIZzWRjSe#d+|bO ziK`m+1{XPoYqb-3{;x$&R5DZBPJ)7IOp~ICz1AI)(!PR2p1z(N@SB~h;6a)EyvSlT zHJ4690?5Y1DMz~Yi`d(@>0Izwox$+ZzD}%l?T~IdF$=pbTw1ACS4FFk?irPiDj-k~ z(SI-Ny0EO-5PWH+945ma$|5mfLZporAQ=B7F7*lCCv*XOE7e`|f+k?p2+`Rx<| zgZwR9kVizI(=&sHF0W)Yek$&l;eo`z9iFS{K3o|!gID5p*A_2E47l9%xR|?nkN~3J z78UJkE2||Ttq%>w@5j!S{DwsEa}zp^gLYiqI@&)v_ghl|uY^-3m#4FeNy=(kHJ|HN zQM=G(g&9kVTKuCYIY@Z6-=$v$y*q9h3lZ#syK5#rHzT3T-?)K|6^@B_9y2S)=Pii8oUeus_7nwP|YNK$hvbYRHW$t z2uE&%x47Ph7)v6mRuAc;Bcu)PUb$H(j0yeq6kK_P{3Q)YiP-#DTvB2LGmTh`iwQNB zjG+AyJ;CCAJ0LPOByHPa8s-_{*nhB)F6jNYVTg*~ICwFR2&2$E)Z0@>)|&Nuzy0;q zx}a0vNZGuSO|3~Ie$CUjtD%8K{06iqcO_b1dxt3Kc>~Hiz5QQ4Q~NR@EHy9lep23n zLcr{;G?Pi_|q*?x>0KioEa1IJCl#ae6p zUMxP+OO9ryXU&%twT)dSM{5)p7tg}XqFyPOs0K<+(sJr@`IM?gECO@>Y;6s1xdvsQ z->Kl&u91XoT^>q-DjYHZ5oz&rzs<%ZRgY5x`2u6FO_=Cc~Qh<34ia4~=$AIoK-~$W)T6Q!945 zQ%#?HS>WnxOjqy*bg|;K2D3meGJD`m$mX>+HTk0!?2-pxNp9K^j3LN5J|FI`UC3C0 zoo+xiGwnWS+k#zJ-@5U6!4zIcLrKhXzUMsgTSbi?rqMjGcC-9uaVhB>Ry%w9-Mj0v zN@Nzrt{r}15ZcGgEZ2`7y0Cxpg;lD1`bHtL(p=zOyF2d(K`vb1J!Afi*STVd!t*8% zGUM8M4F&Nd)bowZab7{eos%zQj|U!~PhtUgi2sds;%+U5oVu`z@p za0G~+Hr4=C#iyEDTXo`(e%7+i#t4uHgNHWyqVl^ylpN{_VX*s`IGev)J!=gQ#`2|T z;kR_0Qlc_sPjg$WO8d*A+_--LeBuzo5^+#3JfX!=KmKC}UW( zYd-2+B7Cc8NAAzo*CBF0nq0{fCx=HzNeP2pNxziR(Q1CXKcVrd%C? zS?Y;IR^J&TPu5H|t$MU1v9Pd?)xB*%X#pV|&jZPUfdPvJZS9!AAP;YklJ-WY=y9NGLY%60 zR_Iw|_?z9Ks>sB3PDZ9A0=Zuek9q!#pGC>X>D=+uEX#P48b?G#9G5)t{~~3$+~Gyu z4Ez~v1lKb)CAJbP6+vH2Txa+Ne1)8xX#gt}2`Jd2IEEYT#*0y*v_ktHp`oF!Zf@`S z$qVW}BjyMIG&D>L7Xh4gd=gso-9d6lQz%WS4qQ)H7aKt-A#?{>^6~DsZ%&b|)}5G} z^V%FFg&ES{@wp=MR$c1{EEkY|mqo9lXuK$&(5TsAa#?bQh{)r=w?b4&=^W7_iF@e^%rm68(-1c{1@a%<%|Kk+=QzywN!VoDA%JyMiU zL{^{e!>5A+HYmvUYX42g;Zmyblk4 z)VRI0)OwQ9WeC#RUb=>JJYs$ZOj2TtiH<)0w1Y#T%FW9=>~hb}&24uY;2bgYLB{hh%?H6w7IATE zgWqh}d(MdEY!SP~hwSup(QDbZ;+B@#ZmNG2+jS)-l=aLl^iQ6or-kcu_7B8S|7R4g zLRxw{Yll>7$5^%qn1d1v@Z-midUX)yBH+VNY7iAwWE6WGojf~Zit2g)V{HmtELt1k zm-fwzY8HeAEZVZy9N#+Fm->7$fSsG0t0TiGQP$PPv;@jKDJ?G_E0Cct!ukH&!}|Hc zU;;gB2|pLX108I(?rofBEtqzn6FM9kCR|kU6imDie4PYq60k9A%azR=3lSXYj{I8!^cl4co0}4vtpHEdH+sLUf z$nK0VN0=w>wt5B9dt_Z^#4}-f|uZd{Re9dsh zjhl-D3}F>M;Yu&z&=j>D(f83_I)~C=@DpwI0^ruQZ()k0-bn&jupK z!w%7^m^!$C)^{$=&dx3@>>YGW$n$_?FPGhF3}N3kh}@+m)CmAR8_pb*Y7of!_wUiY z5N8idQg+DvRlJkK4)uxX#f%=oDWZ&_+RQ{GrhYdm;kVm25HOjipO8it9Vy(fL0R(z+?PV7p + + + NORA's viewer. A medical image viewer and annotation tool. + + + HistoryDatasetAssociation + nrrd + stl + nii1 + nii1.gz + gii + gii.gz + png + jpg + dataset_id + + + + dataset_id + + + + svg + both + + diff --git a/config/plugins/visualizations/nora/package.json b/config/plugins/visualizations/nora/package.json new file mode 100644 index 00000000000..2e42927e715 --- /dev/null +++ b/config/plugins/visualizations/nora/package.json @@ -0,0 +1,18 @@ +{ + "name": "nora", + "version": "1.1.0", + "keywords": [ + "galaxy", + "visualization" + ], + "description": "nora", + "main": "index.js", + "scripts": { + "build": "cpy --cwd='node_modules/@reisertm/nora_galaxy_plugin' --parents './' '../../../static/dist'" + }, + "license": "MIT", + "dependencies": { + "cpy-cli": "^3.1.1", + "@reisertm/nora_galaxy_plugin": "^1.1.6" + } +} From 8afa19785686efa9a88e9fddfea68fc757014622 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 9 Jan 2021 15:15:28 +0100 Subject: [PATCH 030/208] new config --- .../visualizations/nora/__static/logo.png | Bin 5502 -> 0 bytes .../nora/__static/plugin_build_hash.txt | 1 - .../plugins/visualizations/nora/config/nora.xml | 2 +- config/plugins/visualizations/nora/package.json | 1 + 4 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 config/plugins/visualizations/nora/__static/logo.png delete mode 100644 config/plugins/visualizations/nora/__static/plugin_build_hash.txt diff --git a/config/plugins/visualizations/nora/__static/logo.png b/config/plugins/visualizations/nora/__static/logo.png deleted file mode 100644 index 987efb0a6d24be06ce96ba05ba5d4e8fa707363f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5502 zcmb_g^;Z;J6J9!&5?8t#K|&;?L0~CCI>nX7pu4+!>6Ff8L2`j5q(mg7J49NfmtMYo z|A%kRx$(=LGxy$^XJ($5H(Fp)Vme{~06?mys-%lqyD%ez5F0Ze?Ukiq79wX=6E^^W ztpC4(mCi>-4*)z7Qd3gU_s%=acW~ANw+xvX&3+ik;1v9x+iAJr-AgZ)HGl$ODO&V& z@{1Dr{IQ|^9^P3k1W(;Chz;jYe~O>V#+uAJ{S$=sje~+Rm4pHX5gNpgKM56{XX$kM z%4u?oJPM1UXvvtpNbg8Tbdw9Xc}!m#-_5MF-sVj)#1;d9ahDX8jF1x};KmV`;F8(o^9iTWrH!yFn3fXl|xzQJ5CKoT0bw%uB^P?L`pMhb5JU)s60*oQCeEs z+1VK%Pa<15-UgKo4ZEB2B3IVof(}E3vNXA*-->2y-l7hr+;@{lZF3=d*dY+q--S48 zK>-1Q?Gx|#wjHVCbTVZfc6N5j{6r45=`Qc`=4Own^4A}lb5ds6*x3txwzIcJGUJrk zROrs}vefOYt(9tKo(s^$7VkTGiPrcvvzN=W%JFjX^QrZ2Jr~&!p&>DA`BYU~Yh5Y| zkT*7-Uu^!E$#3#)!#V2%RG}W4IBGj(|H8JBc5vR3(`k9n@sMF+KLIO4y{7qF{V zDDTmbR4H+>w?SH3%d$!@};o7I^!4T6TE|B4q!9j)r$h z)pc&;M+vo47XPfUpbqGb*GB=-%BjcfoR%AK>MHFn*(JFbznes3i*t@pKS$NJDYXXi zpaY9il~cPU!NuYJl~o@V6&3aM^=)i8_1H{2@;P{u6OG8shRo^&zbD+HCpA7gT ziUHwYy}iA_*BV&&p{}~ys%6oVy$q|M9DH(eV%czYx;2swl@7RZ_xAQSHKo)0 ztVO|JY9&Oo?lHCatNYSEVMpvWRpUTLb}0a0-sJ)>Y92u%r8$IT-v8J=GNixWO4$!8 zXj`oLC52+}EkRVlI@;Cp>8P0)cKuasz-77@DAa0Kz*0%gG3JgOT>r`Qd9KeBa4s{L zq4x>^p!Lo%gn{5`U2|fhy;$y!L)3*4eJpm+Lqqj%(;B{!RfUy|06Sj-52hvm>#fWu z#CU0N=VXrSSu-b5ad8ohDhiFL;>zA$1^71Z>(4UD8B{fgo_=xw`A1ZLf6dF98i+0< zrk&@I4P%g-r6dC;Bvp)(Ao83t4L*@Ix-Guy{R%!OvcykHp#iR$F@;%Gv^##``QY8Z zyYSBlzZoAOZ|YxK!;R9xy$jycD**?2$?<)4JFr-Tp=6V6@$m4nv$KUrzm*u)$lago z*)El0sqbOp)TvvBfh>x;C!&@5xD5cX09;mGzjU0P$rvAD2*^5`*_FJFWTl5A7s#v( zEiA|uSfmNa7%Jg#yG|3hJTD{6@L;t^C6zA!egD;j<{o+9pRq+JXFU}t3GvC zD7y`1M}iR_I(d+R%x>YUjajSXMX>{+z}t5Btsm)mc?`C>5d_)pDk|oIHm|7;GLRKZ zk_3N^7X6|;8EUV_zy8G&(cRhNC~R|gdvkD2-~GvPFpIB<(#1zDBezv4TwQ_z@baVA zbe^BOy1M7ls*HpLp#%AFSHLZWI>AD)jS!7F(!J^Hu#UKAmyF(=Naf~PTad9Vbo@+c z)5JI{C!JvnW>iF}(~0fb%47F|q?9vh$gIgerFG2v&oaWTudiN2WKe;f5jy5w zKx4zGilO)_9)Lq>_~HCk*C%?uiZ)XtEz>$Z(|YRcvxAFd{K9$K?z}3me`8r$+4-GI z*up1SpHq&_QN>Ud$$#R5qNbUQlga#_L~-ohJ9#QEmCm)G8LKYH5}4K|fYkSJOB>rhMVXcsWS~#c%gj%!tC3 zwePnCl7Y506AGaD6pT#OgqeiiKkr`hsW&QPuaZBx;MlXCh0<^a`CRiS(LdT89v^pd zaCrZ%mq6dhNMwmc+P-u%khWfeZzC7lya_+^S*D%(4FG`f67|KjI1<9q_GvQ9Ms!ay)YnTN7Na#R%JY!6xM&LHMx`ez?#RW)sx1;6<=o8_(Adfl} zs;3^^%)RsciJrK!fC)l^831XS*R+l6zjd@Fd72(OzVk|@p?7L_Hd0BFa9fhe>Ps8$ ziMr2;-kBT5QQL4?*9ok-!1qUuWg?TC06_kI-;m(TwTF-PhjotDT$1yTWRs*^Fc5-` zjqNYIvbTRq0a|CLIF}U4&ByKK9nt0;@uFOCN9+|$@qXu-N!Sl{kIZzWRjSe#d+|bO ziK`m+1{XPoYqb-3{;x$&R5DZBPJ)7IOp~ICz1AI)(!PR2p1z(N@SB~h;6a)EyvSlT zHJ4690?5Y1DMz~Yi`d(@>0Izwox$+ZzD}%l?T~IdF$=pbTw1ACS4FFk?irPiDj-k~ z(SI-Ny0EO-5PWH+945ma$|5mfLZporAQ=B7F7*lCCv*XOE7e`|f+k?p2+`Rx<| zgZwR9kVizI(=&sHF0W)Yek$&l;eo`z9iFS{K3o|!gID5p*A_2E47l9%xR|?nkN~3J z78UJkE2||Ttq%>w@5j!S{DwsEa}zp^gLYiqI@&)v_ghl|uY^-3m#4FeNy=(kHJ|HN zQM=G(g&9kVTKuCYIY@Z6-=$v$y*q9h3lZ#syK5#rHzT3T-?)K|6^@B_9y2S)=Pii8oUeus_7nwP|YNK$hvbYRHW$t z2uE&%x47Ph7)v6mRuAc;Bcu)PUb$H(j0yeq6kK_P{3Q)YiP-#DTvB2LGmTh`iwQNB zjG+AyJ;CCAJ0LPOByHPa8s-_{*nhB)F6jNYVTg*~ICwFR2&2$E)Z0@>)|&Nuzy0;q zx}a0vNZGuSO|3~Ie$CUjtD%8K{06iqcO_b1dxt3Kc>~Hiz5QQ4Q~NR@EHy9lep23n zLcr{;G?Pi_|q*?x>0KioEa1IJCl#ae6p zUMxP+OO9ryXU&%twT)dSM{5)p7tg}XqFyPOs0K<+(sJr@`IM?gECO@>Y;6s1xdvsQ z->Kl&u91XoT^>q-DjYHZ5oz&rzs<%ZRgY5x`2u6FO_=Cc~Qh<34ia4~=$AIoK-~$W)T6Q!945 zQ%#?HS>WnxOjqy*bg|;K2D3meGJD`m$mX>+HTk0!?2-pxNp9K^j3LN5J|FI`UC3C0 zoo+xiGwnWS+k#zJ-@5U6!4zIcLrKhXzUMsgTSbi?rqMjGcC-9uaVhB>Ry%w9-Mj0v zN@Nzrt{r}15ZcGgEZ2`7y0Cxpg;lD1`bHtL(p=zOyF2d(K`vb1J!Afi*STVd!t*8% zGUM8M4F&Nd)bowZab7{eos%zQj|U!~PhtUgi2sds;%+U5oVu`z@p za0G~+Hr4=C#iyEDTXo`(e%7+i#t4uHgNHWyqVl^ylpN{_VX*s`IGev)J!=gQ#`2|T z;kR_0Qlc_sPjg$WO8d*A+_--LeBuzo5^+#3JfX!=KmKC}UW( zYd-2+B7Cc8NAAzo*CBF0nq0{fCx=HzNeP2pNxziR(Q1CXKcVrd%C? zS?Y;IR^J&TPu5H|t$MU1v9Pd?)xB*%X#pV|&jZPUfdPvJZS9!AAP;YklJ-WY=y9NGLY%60 zR_Iw|_?z9Ks>sB3PDZ9A0=Zuek9q!#pGC>X>D=+uEX#P48b?G#9G5)t{~~3$+~Gyu z4Ez~v1lKb)CAJbP6+vH2Txa+Ne1)8xX#gt}2`Jd2IEEYT#*0y*v_ktHp`oF!Zf@`S z$qVW}BjyMIG&D>L7Xh4gd=gso-9d6lQz%WS4qQ)H7aKt-A#?{>^6~DsZ%&b|)}5G} z^V%FFg&ES{@wp=MR$c1{EEkY|mqo9lXuK$&(5TsAa#?bQh{)r=w?b4&=^W7_iF@e^%rm68(-1c{1@a%<%|Kk+=QzywN!VoDA%JyMiU zL{^{e!>5A+HYmvUYX42g;Zmyblk4 z)VRI0)OwQ9WeC#RUb=>JJYs$ZOj2TtiH<)0w1Y#T%FW9=>~hb}&24uY;2bgYLB{hh%?H6w7IATE zgWqh}d(MdEY!SP~hwSup(QDbZ;+B@#ZmNG2+jS)-l=aLl^iQ6or-kcu_7B8S|7R4g zLRxw{Yll>7$5^%qn1d1v@Z-midUX)yBH+VNY7iAwWE6WGojf~Zit2g)V{HmtELt1k zm-fwzY8HeAEZVZy9N#+Fm->7$fSsG0t0TiGQP$PPv;@jKDJ?G_E0Cct!ukH&!}|Hc zU;;gB2|pLX108I(?rofBEtqzn6FM9kCR|kU6imDie4PYq60k9A%azR=3lSXYj{I8!^cl4co0}4vtpHEdH+sLUf z$nK0VN0=w>wt5B9dt_Z^#4}-f|uZd{Re9dsh zjhl-D3}F>M;Yu&z&=j>D(f83_I)~C=@DpwI0^ruQZ()k0-bn&jupK z!w%7^m^!$C)^{$=&dx3@>>YGW$n$_?FPGhF3}N3kh}@+m)CmAR8_pb*Y7of!_wUiY z5N8idQg+DvRlJkK4)uxX#f%=oDWZ&_+RQ{GrhYdm;kVm25HOjipO8it9Vy(fL0R(z+?PV7p svg - both + optional diff --git a/config/plugins/visualizations/nora/package.json b/config/plugins/visualizations/nora/package.json index 2e42927e715..8e8b9ed0cb5 100644 --- a/config/plugins/visualizations/nora/package.json +++ b/config/plugins/visualizations/nora/package.json @@ -13,6 +13,7 @@ "license": "MIT", "dependencies": { "cpy-cli": "^3.1.1", + "underscore": "^1.8.3", "@reisertm/nora_galaxy_plugin": "^1.1.6" } } From a96570eab65162e80b9f756516e7fee5a4f05134 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sun, 10 Jan 2021 18:05:06 -0500 Subject: [PATCH 031/208] Add missing id keys to built-in tours Pydantic models (currently added to tours) require explicit specification of model fields. Having an "id" field as required seems right from a design standpoint. However, if there is a reason for an "id" value to be included in the galaxy_ui tour but not in the other tours, this field can be made optional: `id: Optional[str] = None` --- config/plugins/tours/core.history.yaml | 1 + config/plugins/tours/core.scratchbook.yaml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/plugins/tours/core.history.yaml b/config/plugins/tours/core.history.yaml index 7264e1538f3..f0ce1e3b28a 100644 --- a/config/plugins/tours/core.history.yaml +++ b/config/plugins/tours/core.history.yaml @@ -1,3 +1,4 @@ +id: galaxy_history name: History Introduction description: A detailed introduction to the Galaxy History title_default: "Galaxy History Introduction" diff --git a/config/plugins/tours/core.scratchbook.yaml b/config/plugins/tours/core.scratchbook.yaml index 6b878772257..74e892eea62 100644 --- a/config/plugins/tours/core.scratchbook.yaml +++ b/config/plugins/tours/core.scratchbook.yaml @@ -1,6 +1,7 @@ +id: galaxy_scratchbook name: Scratchbook - Introduction -title_default: "Scratchbook Introduction" description: "An introduction on how to display multiple datasets and visualizations next to each other." +title_default: "Scratchbook Introduction" tags: - "core" - "UI" From 0cc263214222204c005cfde8afc10df26418f67b Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sun, 10 Jan 2021 18:21:44 -0500 Subject: [PATCH 032/208] Add test, assertions for tour API testing --- lib/galaxy_test/api/test_tours.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/galaxy_test/api/test_tours.py b/lib/galaxy_test/api/test_tours.py index b15a1e71442..6c2ecde6836 100644 --- a/lib/galaxy_test/api/test_tours.py +++ b/lib/galaxy_test/api/test_tours.py @@ -9,9 +9,20 @@ class TourApiTestCase(ApiTestCase): tours = response.json() tour_keys = map(lambda t: t["id"], tours) assert "core.history" in tour_keys + for tour in tours: + self._assert_has_keys(tour, "id", "name", "description", "tags") def test_show(self): response = self._get("tours/core.history") self._assert_status_code_is(response, 200) tour = response.json() + self._assert_tour(tour) + + def test_update(self): + response = self._post("tours/core.history", admin=True) + self._assert_status_code_is(response, 200) + tour = response.json() + self._assert_tour(tour) + + def _assert_tour(self, tour): self._assert_has_keys(tour, "name", "description", "title_default", "steps") From 82e3db5123a35e0dfffca69d91de0804c01d359a Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sun, 10 Jan 2021 19:23:48 -0500 Subject: [PATCH 033/208] Add FastAPI routes, Pydantic models to tours --- lib/galaxy/tours/__init__.py | 42 ++++++++++++++++++++++---- lib/galaxy/webapps/galaxy/api/tours.py | 37 +++++++++++++++++++++++ lib/galaxy/webapps/galaxy/fast_app.py | 10 ++++-- 3 files changed, 81 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/tours/__init__.py b/lib/galaxy/tours/__init__.py index 2239caf4804..171f8f3393c 100644 --- a/lib/galaxy/tours/__init__.py +++ b/lib/galaxy/tours/__init__.py @@ -3,14 +3,39 @@ This module manages loading/etc of Galaxy interactive tours. """ import logging import os +from typing import List, Optional import yaml +from pydantic import BaseModel from galaxy import util log = logging.getLogger(__name__) +class Tour(BaseModel): + id: str + name: str + description: str + tags: List[str] + + +class TourStep(BaseModel): + title: Optional[str] = None + content: Optional[str] = None + element: Optional[str] = None + placement: Optional[str] = None + preclick: Optional[list] = None + postclick: Optional[list] = None + textinsert: Optional[str] = None + backdrop: Optional[bool] = None + + +class TourDetails(Tour): + title_default: Optional[str] = None + steps: List[TourStep] + + def tour_loader(contents_dict): # Some of this can be done on the clientside. Maybe even should? title_default = contents_dict.get('title_default', None) @@ -31,12 +56,17 @@ class ToursRegistry: self.tour_directories = util.config_directories_from_setting(tour_directories) self.load_tours() - def tours_by_id_with_description(self): - return [{'id': k, - 'description': self.tours[k].get('description', None), - 'name': self.tours[k].get('name', None), - 'tags': self.tours[k].get('tags', None)} - for k in self.tours.keys()] + def tours_by_id_with_description(self) -> List[Tour]: + tours = [] + for k in self.tours.keys(): + tourdata = { + 'id': k, + 'description': self.tours[k].get('description'), + 'name': self.tours[k].get('name'), + 'tags': self.tours[k].get('tags') + } + tours.append(Tour(**tourdata)) + return tours def load_tour(self, tour_id): for tour_dir in self.tour_directories: diff --git a/lib/galaxy/webapps/galaxy/api/tours.py b/lib/galaxy/webapps/galaxy/api/tours.py index 2837959644c..e7847379254 100644 --- a/lib/galaxy/webapps/galaxy/api/tours.py +++ b/lib/galaxy/webapps/galaxy/api/tours.py @@ -2,17 +2,54 @@ API Controller providing Galaxy Tours """ import logging +from typing import List +from fastapi import Depends +from fastapi_utils.cbv import cbv +from fastapi_utils.inferring_router import InferringRouter as APIRouter + +from galaxy.app import UniverseApplication +from galaxy.tours import ( + Tour, + TourDetails, +) from galaxy.web import ( expose_api_anonymous_and_sessionless, legacy_expose_api, require_admin ) from galaxy.webapps.base.controller import BaseAPIController +from . import ( + get_admin_user, + get_app, +) log = logging.getLogger(__name__) +router = APIRouter(tags=['tours']) + + +@cbv(router) +class FastAPITours: + app: UniverseApplication = Depends(get_app) + + @router.get('/api/tours') + def index(self) -> List[Tour]: + """Return list of available tours.""" + return self.app.tour_registry.tours_by_id_with_description() + + @router.get('/api/tours/{tour_id}') + def show(self, tour_id: str) -> TourDetails: + """Return a tour definition.""" + return self.app.tour_registry.tour_contents(tour_id) + + @router.post('/api/tours/{tour_id}', dependencies=[Depends(get_admin_user)]) + def update_tour(self, tour_id: str) -> TourDetails: + """Return a tour definition.""" + return self.app.tour_registry.load_tour(tour_id) + + class ToursController(BaseAPIController): def __init__(self, app): diff --git a/lib/galaxy/webapps/galaxy/fast_app.py b/lib/galaxy/webapps/galaxy/fast_app.py index 64ad1e29329..069011c178d 100644 --- a/lib/galaxy/webapps/galaxy/fast_app.py +++ b/lib/galaxy/webapps/galaxy/fast_app.py @@ -16,6 +16,10 @@ api_tags_metadata = [ "name": "licenses", "description": "Operations with [SPDX licenses](https://spdx.org/licenses/).", }, + { + "name": "tours", + "description": "Operations with interactive tours.", + }, ] @@ -49,12 +53,14 @@ def initialize_fast_app(gx_app): from galaxy.webapps.galaxy.api import ( job_lock, jobs, + licenses, roles, - licenses + tours, ) app.include_router(jobs.router) app.include_router(job_lock.router) - app.include_router(roles.router) app.include_router(licenses.router) + app.include_router(roles.router) + app.include_router(tours.router) app.mount('/', wsgi_handler) return app From 759aca7b2cb72f3018f323befeb74e938943051a Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 11 Jan 2021 02:28:22 -0500 Subject: [PATCH 034/208] Make a TourList model for correct serialization --- lib/galaxy/tours/__init__.py | 10 +++++++--- lib/galaxy/webapps/galaxy/api/tours.py | 5 ++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/tours/__init__.py b/lib/galaxy/tours/__init__.py index 171f8f3393c..8d29605f3f1 100644 --- a/lib/galaxy/tours/__init__.py +++ b/lib/galaxy/tours/__init__.py @@ -6,7 +6,7 @@ import os from typing import List, Optional import yaml -from pydantic import BaseModel +from pydantic import BaseModel, parse_obj_as from galaxy import util @@ -20,6 +20,10 @@ class Tour(BaseModel): tags: List[str] +class TourList(BaseModel): + __root__: List[Tour] = [] + + class TourStep(BaseModel): title: Optional[str] = None content: Optional[str] = None @@ -56,7 +60,7 @@ class ToursRegistry: self.tour_directories = util.config_directories_from_setting(tour_directories) self.load_tours() - def tours_by_id_with_description(self) -> List[Tour]: + def tours_by_id_with_description(self) -> TourList: tours = [] for k in self.tours.keys(): tourdata = { @@ -66,7 +70,7 @@ class ToursRegistry: 'tags': self.tours[k].get('tags') } tours.append(Tour(**tourdata)) - return tours + return parse_obj_as(TourList, tours) def load_tour(self, tour_id): for tour_dir in self.tour_directories: diff --git a/lib/galaxy/webapps/galaxy/api/tours.py b/lib/galaxy/webapps/galaxy/api/tours.py index e7847379254..71a7b57fbf9 100644 --- a/lib/galaxy/webapps/galaxy/api/tours.py +++ b/lib/galaxy/webapps/galaxy/api/tours.py @@ -2,7 +2,6 @@ API Controller providing Galaxy Tours """ import logging -from typing import List from fastapi import Depends from fastapi_utils.cbv import cbv @@ -10,8 +9,8 @@ from fastapi_utils.inferring_router import InferringRouter as APIRouter from galaxy.app import UniverseApplication from galaxy.tours import ( - Tour, TourDetails, + TourList, ) from galaxy.web import ( expose_api_anonymous_and_sessionless, @@ -35,7 +34,7 @@ class FastAPITours: app: UniverseApplication = Depends(get_app) @router.get('/api/tours') - def index(self) -> List[Tour]: + def index(self) -> TourList: """Return list of available tours.""" return self.app.tour_registry.tours_by_id_with_description() From dbae0339f9082c5ba4ddf732788648d1fb6af962 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 11 Jan 2021 02:35:13 -0500 Subject: [PATCH 035/208] Refactor tour loading (minor) - Do not call tour_loader() twice - Do not return a value from a function that modifies the value of its argument in place - Rename function to reflect what it does --- lib/galaxy/tours/__init__.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/tours/__init__.py b/lib/galaxy/tours/__init__.py index 8d29605f3f1..7297b60cde9 100644 --- a/lib/galaxy/tours/__init__.py +++ b/lib/galaxy/tours/__init__.py @@ -40,7 +40,7 @@ class TourDetails(Tour): steps: List[TourStep] -def tour_loader(contents_dict): +def load_steps(contents_dict): # Some of this can be done on the clientside. Maybe even should? title_default = contents_dict.get('title_default', None) for step in contents_dict['steps']: @@ -52,7 +52,6 @@ def tour_loader(contents_dict): step['orphan'] = True if title_default and 'title' not in step: step['title'] = title_default - return contents_dict class ToursRegistry: @@ -107,9 +106,9 @@ class ToursRegistry: tour_id = os.path.splitext(filename)[0] try: with open(tour_path) as handle: - conf = yaml.safe_load(handle) - tour = tour_loader(conf) - self.tours[tour_id] = tour_loader(conf) + tour = yaml.safe_load(handle) + load_steps(tour) + self.tours[tour_id] = tour log.info("Loaded tour '%s'" % tour_id) return tour except OSError: From ca5516bd26323f0d8259ec67bd46f4cea3bf2292 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 11 Jan 2021 02:49:09 -0500 Subject: [PATCH 036/208] Simplify parsing pydantic models --- lib/galaxy/tours/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tours/__init__.py b/lib/galaxy/tours/__init__.py index 7297b60cde9..2c9c7dce7e9 100644 --- a/lib/galaxy/tours/__init__.py +++ b/lib/galaxy/tours/__init__.py @@ -68,7 +68,7 @@ class ToursRegistry: 'name': self.tours[k].get('name'), 'tags': self.tours[k].get('tags') } - tours.append(Tour(**tourdata)) + tours.append(tourdata) return parse_obj_as(TourList, tours) def load_tour(self, tour_id): From 6021a10961fbdfeeb69d709a159aeba484ebccd5 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 11 Jan 2021 03:18:57 -0500 Subject: [PATCH 037/208] Move pydantic models to schema/ --- lib/galaxy/schema/__init__.py | 37 +++++++++++++++++++++++--- lib/galaxy/tours/__init__.py | 32 +++------------------- lib/galaxy/webapps/galaxy/api/tours.py | 2 +- 3 files changed, 38 insertions(+), 33 deletions(-) diff --git a/lib/galaxy/schema/__init__.py b/lib/galaxy/schema/__init__.py index 0533680cb97..e6130a8f84b 100644 --- a/lib/galaxy/schema/__init__.py +++ b/lib/galaxy/schema/__init__.py @@ -1,13 +1,44 @@ -import typing +from typing import ( + Dict, + List, + Optional, +) from pydantic import BaseModel class BootstrapAdminUser(BaseModel): id = 0 - email: typing.Optional[str] = None - preferences: typing.Dict[str, str] = {} + email: Optional[str] = None + preferences: Dict[str, str] = {} bootstrap_admin_user = True def all_roles(*args) -> list: return [] + + +class Tour(BaseModel): + id: str + name: str + description: str + tags: List[str] + + +class TourList(BaseModel): + __root__: List[Tour] = [] + + +class TourStep(BaseModel): + title: Optional[str] = None + content: Optional[str] = None + element: Optional[str] = None + placement: Optional[str] = None + preclick: Optional[list] = None + postclick: Optional[list] = None + textinsert: Optional[str] = None + backdrop: Optional[bool] = None + + +class TourDetails(Tour): + title_default: Optional[str] = None + steps: List[TourStep] diff --git a/lib/galaxy/tours/__init__.py b/lib/galaxy/tours/__init__.py index 2c9c7dce7e9..7acf602674e 100644 --- a/lib/galaxy/tours/__init__.py +++ b/lib/galaxy/tours/__init__.py @@ -3,43 +3,17 @@ This module manages loading/etc of Galaxy interactive tours. """ import logging import os -from typing import List, Optional import yaml -from pydantic import BaseModel, parse_obj_as +from pydantic import parse_obj_as from galaxy import util +from galaxy.schema import TourList + log = logging.getLogger(__name__) -class Tour(BaseModel): - id: str - name: str - description: str - tags: List[str] - - -class TourList(BaseModel): - __root__: List[Tour] = [] - - -class TourStep(BaseModel): - title: Optional[str] = None - content: Optional[str] = None - element: Optional[str] = None - placement: Optional[str] = None - preclick: Optional[list] = None - postclick: Optional[list] = None - textinsert: Optional[str] = None - backdrop: Optional[bool] = None - - -class TourDetails(Tour): - title_default: Optional[str] = None - steps: List[TourStep] - - def load_steps(contents_dict): # Some of this can be done on the clientside. Maybe even should? title_default = contents_dict.get('title_default', None) diff --git a/lib/galaxy/webapps/galaxy/api/tours.py b/lib/galaxy/webapps/galaxy/api/tours.py index 71a7b57fbf9..835d778838a 100644 --- a/lib/galaxy/webapps/galaxy/api/tours.py +++ b/lib/galaxy/webapps/galaxy/api/tours.py @@ -8,7 +8,7 @@ from fastapi_utils.cbv import cbv from fastapi_utils.inferring_router import InferringRouter as APIRouter from galaxy.app import UniverseApplication -from galaxy.tours import ( +from galaxy.schema import ( TourDetails, TourList, ) From 594518c60846e037991a0230b133832e2f566e81 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 11 Jan 2021 13:53:17 +0100 Subject: [PATCH 038/208] Add missing tests --- lib/galaxy_test/api/test_datatypes.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/galaxy_test/api/test_datatypes.py b/lib/galaxy_test/api/test_datatypes.py index bc174af2448..0634412a07c 100644 --- a/lib/galaxy_test/api/test_datatypes.py +++ b/lib/galaxy_test/api/test_datatypes.py @@ -84,6 +84,18 @@ class DatatypesApiTestCase(ApiTestCase): self._assert_status_code_is(response, 200) assert converters == len(response.json()) + def test_edamp_formats(self): + response = self._get("datatypes/edam_formats") + self._assert_status_code_is(response, 200) + edam_formats = response.json() + assert isinstance(edam_formats, dict) + + def test_edam_data(self): + response = self._get("datatypes/edam_data") + self._assert_status_code_is(response, 200) + edam_data = response.json() + assert isinstance(edam_data, dict) + def _index_datatypes(self, data=None): data = data or {} response = self._get("datatypes", data=data) From 9d107e6943506e1d03319ad32024eb976f66301b Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Mon, 11 Jan 2021 08:27:59 -0500 Subject: [PATCH 039/208] Added check for runtime value. --- lib/galaxy/tools/parameters/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 5a4101e2f0a..d2bab941599 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1349,7 +1349,7 @@ class ColumnListParameter(SelectToolParameter): # Use representative dataset if a dataset collection is parsed if isinstance(dataset, trans.app.model.HistoryDatasetCollectionAssociation): dataset = dataset.to_hda_representative() - if not dataset.has_data(): + if not is_runtime_value(dataset) and not dataset.has_data(): return True return False From a7d8d67c56ec307c0e7b33e65b5e24eeb70f8c03 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 11 Jan 2021 10:19:04 -0500 Subject: [PATCH 040/208] Move tour pydantic models to their own module --- lib/galaxy/schema/__init__.py | 28 ---------------------- lib/galaxy/tours/__init__.py | 2 +- lib/galaxy/tours/schema.py | 33 ++++++++++++++++++++++++++ lib/galaxy/webapps/galaxy/api/tours.py | 2 +- 4 files changed, 35 insertions(+), 30 deletions(-) create mode 100644 lib/galaxy/tours/schema.py diff --git a/lib/galaxy/schema/__init__.py b/lib/galaxy/schema/__init__.py index e6130a8f84b..174c87af56d 100644 --- a/lib/galaxy/schema/__init__.py +++ b/lib/galaxy/schema/__init__.py @@ -1,6 +1,5 @@ from typing import ( Dict, - List, Optional, ) @@ -15,30 +14,3 @@ class BootstrapAdminUser(BaseModel): def all_roles(*args) -> list: return [] - - -class Tour(BaseModel): - id: str - name: str - description: str - tags: List[str] - - -class TourList(BaseModel): - __root__: List[Tour] = [] - - -class TourStep(BaseModel): - title: Optional[str] = None - content: Optional[str] = None - element: Optional[str] = None - placement: Optional[str] = None - preclick: Optional[list] = None - postclick: Optional[list] = None - textinsert: Optional[str] = None - backdrop: Optional[bool] = None - - -class TourDetails(Tour): - title_default: Optional[str] = None - steps: List[TourStep] diff --git a/lib/galaxy/tours/__init__.py b/lib/galaxy/tours/__init__.py index 7acf602674e..cb77527700e 100644 --- a/lib/galaxy/tours/__init__.py +++ b/lib/galaxy/tours/__init__.py @@ -8,7 +8,7 @@ import yaml from pydantic import parse_obj_as from galaxy import util -from galaxy.schema import TourList +from galaxy.tours.schema import TourList log = logging.getLogger(__name__) diff --git a/lib/galaxy/tours/schema.py b/lib/galaxy/tours/schema.py new file mode 100644 index 00000000000..b738d762a5d --- /dev/null +++ b/lib/galaxy/tours/schema.py @@ -0,0 +1,33 @@ +from typing import ( + List, + Optional, +) + +from pydantic import BaseModel + + +class Tour(BaseModel): + id: str + name: str + description: str + tags: List[str] + + +class TourList(BaseModel): + __root__: List[Tour] = [] + + +class TourStep(BaseModel): + title: Optional[str] = None + content: Optional[str] = None + element: Optional[str] = None + placement: Optional[str] = None + preclick: Optional[list] = None + postclick: Optional[list] = None + textinsert: Optional[str] = None + backdrop: Optional[bool] = None + + +class TourDetails(Tour): + title_default: Optional[str] = None + steps: List[TourStep] diff --git a/lib/galaxy/webapps/galaxy/api/tours.py b/lib/galaxy/webapps/galaxy/api/tours.py index 835d778838a..f8e9f97228d 100644 --- a/lib/galaxy/webapps/galaxy/api/tours.py +++ b/lib/galaxy/webapps/galaxy/api/tours.py @@ -8,7 +8,7 @@ from fastapi_utils.cbv import cbv from fastapi_utils.inferring_router import InferringRouter as APIRouter from galaxy.app import UniverseApplication -from galaxy.schema import ( +from galaxy.tours.schema import ( TourDetails, TourList, ) From d968b6beae22006f38bd59ac8ce0dcd6b6980113 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 11 Jan 2021 17:48:11 +0100 Subject: [PATCH 041/208] Add datatypes schemas --- lib/galaxy/datatypes/schema.py | 118 +++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 lib/galaxy/datatypes/schema.py diff --git a/lib/galaxy/datatypes/schema.py b/lib/galaxy/datatypes/schema.py new file mode 100644 index 00000000000..1e82c249f1a --- /dev/null +++ b/lib/galaxy/datatypes/schema.py @@ -0,0 +1,118 @@ +from typing import Dict, List, Optional + +from pydantic import ( + BaseModel, + Field, + HttpUrl +) + + +class CompositeFileInfo(BaseModel): + name: str = Field( + ..., # Mark this field as required + title="Name", + description="The name of this composite file" + ) + optional: bool = Field( + title="Optional", + description="" # TODO add description + ) + mimetype: Optional[str] = Field( + title="MIME type", + description="The MIME type of this file" + ) + description: Optional[str] = Field( + title="Description", + description="Summary description of the purpouse of this file" + ) + substitute_name_with_metadata: Optional[str] = Field( + title="Substitute name with metadata", + description="" # TODO add description + ) + is_binary: bool = Field( + title="Is binary", + description="Whether this file is a binary file" + ) + to_posix_lines: bool = Field( + title="To posix lines", + description="" # TODO add description + ) + space_to_tab: bool = Field( + title="Spaces to tabulation", + description="" # TODO add description + ) + + +class DatatypeDetails(BaseModel): + extension: str = Field( + ..., # Mark this field as required + title="Extension", + description="The data type’s Dataset file extension", + example="bed" + ) + description: Optional[str] = Field( + title="Description", + description="A summary description for this data type" + ) + description_url: Optional[HttpUrl] = Field( + title="Description URL", + description="The URL to a detailed description for this datatype", + example="https://wiki.galaxyproject.org/Learn/Datatypes#Bed" + ) + display_in_upload: bool = Field( + default=False, + title="Display in upload", + description="If True, the associated file extension will be displayed in the `File Format` select list in the `Upload File from your computer` tool in the `Get Data` tool section of the tool panel" + ) + composite_files: Optional[List[CompositeFileInfo]] = Field( + default=None, + title="Composite files", + description="A collection of files composing this data type" + ) + + +class DatatypesMap(BaseModel): + ext_to_class_name: Dict[str, str] = Field( + ..., # Mark this field as required + title="Extension Map", + description="Dictionary mapping datatype's extensions with implementation classes", + ) + class_to_classes: Dict[str, Dict[str, bool]] = Field( + ..., # Mark this field as required + title="Classes Map", + description="Dictionary mapping datatype's classes with their base classes", + ) + + +class DatatypesCombinedMap(BaseModel): + datatypes: List[str] = Field( + ..., # Mark this field as required + title="Datatypes", + description="List of datatypes extensions", + ) + datatypes_mapping: DatatypesMap = Field( + ..., # Mark this field as required + title="Datatypes Mapping", + description="Dictionaries for mapping datatype's extensions/classes with their implementation classes", + ) + + +class DatatypeConverter(BaseModel): + source: str = Field( + ..., # Mark this field as required + title="Source", + description="Source type for conversion", + example="bam", + ) + target: str = Field( + ..., # Mark this field as required + title="Target", + description="Target type for conversion", + example="bai", + ) + tool_id: str = Field( + ..., # Mark this field as required + title="Tool identifier", + description="The converter tool identifier", + example="CONVERTER_Bam_Bai_0", + ) From 82ae2958b707fa50508128e3e2ec6d6fd104067c Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 11 Jan 2021 17:50:40 +0100 Subject: [PATCH 042/208] Add datatypes routes with documentation --- lib/galaxy/webapps/galaxy/api/datatypes.py | 187 +++++++++++++++++++++ 1 file changed, 187 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/api/datatypes.py b/lib/galaxy/webapps/galaxy/api/datatypes.py index 3f17f881df6..89f0880b8dc 100644 --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -2,14 +2,201 @@ API operations allowing clients to determine datatype supported by Galaxy. """ import logging +from typing import ( + Dict, + List, + Optional, + Union, +) +from fastapi import ( + Depends, + Query, +) +from fastapi.routing import APIRouter +from fastapi_utils.cbv import cbv + + +from galaxy.app import UniverseApplication from galaxy.datatypes.data import Data +from galaxy.datatypes.registry import Registry +from galaxy.datatypes.schema import ( + DatatypeConverter, + DatatypeDetails, + DatatypesCombinedMap, + DatatypesMap, +) from galaxy.util import asbool from galaxy.web import expose_api_anonymous_and_sessionless from galaxy.webapps.base.controller import BaseAPIController +from . import get_app + log = logging.getLogger(__name__) +router = APIRouter(tags=['datatypes']) + +ExtensionOnlyQueryParam : Optional[bool] = Query( + default=True, + title="Extension only", + description="Whether to return only the datatype's extension rather than the datatype's details", +) + +UploadOnlyQueryParam : Optional[bool] = Query( + default=True, + title="Upload only", + description="Whether to return only datatypes which can be uploaded", +) + + +def get_datatypes_registry(app: UniverseApplication = Depends(get_app)) -> Registry: + return app.datatypes_registry + + +@cbv(router) +class FastAPIDatatypes: + datatypes_registry: Registry = Depends(get_datatypes_registry) + + @router.get('/api/datatypes', + summary="Lists all available data types", + response_model=Union[List[DatatypeDetails], List[str]], + response_description="List of data types") + async def index(self, + extension_only: Optional[bool] = ExtensionOnlyQueryParam, + upload_only: Optional[bool] = UploadOnlyQueryParam + ) -> Union[List[DatatypeDetails], List[str]]: + """Gets the list of all available data types.""" + try: + return self._index(extension_only, upload_only) + except Exception as e: + log.exception('Could not get datat ypes') + raise e + + @router.get('/api/datatypes/mapping', + summary="Returns mappings for data types and their implementing classes", + response_model=DatatypesMap, + response_description="Dictionary to map data types with their classes") + async def mapping(self) -> DatatypesMap: + """Gets mappings for data types.""" + try: + return self._mapping() + except Exception as e: + log.exception('Could not get data type mapping') + raise e + + @router.get('/api/datatypes/types_and_mapping', + summary="Returns all the data types extensions and their mappings", + response_model=DatatypesCombinedMap, + response_description="Dictionary to map data types with their classes") + async def types_and_mapping(self) -> DatatypesCombinedMap: + """Combines the datatype information from (/api/datatypes) and the + mapping information from (/api/datatypes/mapping) into a single + response.""" + return DatatypesCombinedMap(datatypes=self._index(), datatypes_mapping=self._mapping()) + + @router.get('/api/datatypes/sniffers', + summary="Returns the list of all installed sniffers", + response_model=List[str], + response_description="List of datatype sniffers") + async def sniffers(self) -> List[str]: + """Gets the list of all installed data type sniffers.""" + try: + return self._sniffers() + except Exception as e: + log.exception('Could not get sniffers') + raise e + + @router.get('/api/datatypes/converters', + summary="Returns the list of all installed converters", + response_model=List[DatatypeConverter], + response_description="List of all datatype converters") + async def converters(self) -> List[DatatypeConverter]: + """Gets the list of all installed converters.""" + try: + return self._converters() + except Exception as e: + log.exception('Could not get converters') + raise e + + @router.get('/api/datatypes/edam_formats', + summary="Returns a dictionary/map of datatypes and EDAM formats", + response_model=Dict[str, str], + response_description="Dictionary/map of datatypes and EDAM formats") + async def edam_formats(self) -> Dict[str, str]: + """Gets a map of datatypes and their corresponding EDAM formats.""" + try: + return self.datatypes_registry.edam_formats + except Exception as e: + log.exception('Could not get EDAM format mappings') + raise e + + @router.get('/api/datatypes/edam_data', + summary="Returns a dictionary/map of datatypes and EDAM data", + response_model=Dict[str, str], + response_description="Dictionary/map of datatypes and EDAM data") + async def edam_data(self) -> Dict[str, str]: + """Gets a map of datatypes and their corresponding EDAM data.""" + try: + return self.datatypes_registry.edam_data + except Exception as e: + log.exception('Could not get EDAM data mappings') + raise e + + def _index(self, extension_only: Optional[bool] = True, upload_only: Optional[bool] = True) -> Union[List[DatatypeDetails], List[str]]: + if extension_only: + if upload_only: + return self.datatypes_registry.upload_file_formats + else: + return [ext for ext in self.datatypes_registry.datatypes_by_extension] + else: + rval = [] + for datatype_info_dict in self.datatypes_registry.datatype_info_dicts: + if upload_only and not datatype_info_dict.get('display_in_upload'): + continue + rval.append(datatype_info_dict) + return rval + + def _mapping(self) -> DatatypesMap: + ext_to_class_name: Dict[str, str] = {} + classes = [] + for k, v in self.datatypes_registry.datatypes_by_extension.items(): + c = v.__class__ + ext_to_class_name[k] = f"{c.__module__}.{c.__name__}" + classes.append(c) + class_to_classes: Dict[str, Dict[str, bool]] = {} + + def visit_bases(types, cls): + for base in cls.__bases__: + if issubclass(base, Data): + types.add(f"{base.__module__}.{base.__name__}") + visit_bases(types, base) + + for c in classes: + n = f"{c.__module__}.{c.__name__}" + types = {n} + visit_bases(types, c) + class_to_classes[n] = {t: True for t in types} + return DatatypesMap(ext_to_class_name=ext_to_class_name, class_to_classes=class_to_classes) + + def _sniffers(self) -> List[str]: + rval: List[str] = [] + for sniffer_elem in self.datatypes_registry.sniffer_elems: + datatype = sniffer_elem.get('type') + if datatype is not None: + rval.append(datatype) + return rval + + def _converters(self) -> List[DatatypeConverter]: + converters: List[DatatypeConverter] = [] + for (source_type, targets) in self.datatypes_registry.datatype_converters.items(): + for target_type in targets: + converters.append(DatatypeConverter( + source=source_type, + target=target_type, + tool_id=targets[target_type].id, + )) + return converters + class DatatypesController(BaseAPIController): From 4ac0f6f882dbabb6afb174678cdc3e3be5a8686b Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 11 Jan 2021 17:56:02 +0100 Subject: [PATCH 043/208] Include router in API --- lib/galaxy/webapps/galaxy/fast_app.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/fast_app.py b/lib/galaxy/webapps/galaxy/fast_app.py index 64ad1e29329..5fe94519e33 100644 --- a/lib/galaxy/webapps/galaxy/fast_app.py +++ b/lib/galaxy/webapps/galaxy/fast_app.py @@ -12,6 +12,10 @@ from galaxy.web.framework.decorators import ( # https://fastapi.tiangolo.com/tutorial/metadata/#metadata-for-tags api_tags_metadata = [ + { + "name": "datatypes", + "description": "Operations with supported data types.", + }, { "name": "licenses", "description": "Operations with [SPDX licenses](https://spdx.org/licenses/).", @@ -47,11 +51,13 @@ def initialize_fast_app(gx_app): add_exception_handler(app) wsgi_handler = WSGIMiddleware(gx_app) from galaxy.webapps.galaxy.api import ( + datatypes, job_lock, jobs, roles, licenses ) + app.include_router(datatypes.router) app.include_router(jobs.router) app.include_router(job_lock.router) app.include_router(roles.router) From 64ef16a781ebb4d14dde0d4181f6f03e4ee28daf Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 11 Jan 2021 18:02:30 +0100 Subject: [PATCH 044/208] Check additional conditions in new tests --- lib/galaxy_test/api/test_datatypes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy_test/api/test_datatypes.py b/lib/galaxy_test/api/test_datatypes.py index 0634412a07c..e310cf11bbe 100644 --- a/lib/galaxy_test/api/test_datatypes.py +++ b/lib/galaxy_test/api/test_datatypes.py @@ -89,12 +89,14 @@ class DatatypesApiTestCase(ApiTestCase): self._assert_status_code_is(response, 200) edam_formats = response.json() assert isinstance(edam_formats, dict) + assert edam_formats["ab1"] == "format_3000" def test_edam_data(self): response = self._get("datatypes/edam_data") self._assert_status_code_is(response, 200) edam_data = response.json() assert isinstance(edam_data, dict) + assert edam_data["ab1"] == "data_0924" def _index_datatypes(self, data=None): data = data or {} From d6682f0c0c510ea55f6251165276f62d80686629 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 11 Jan 2021 18:15:25 +0100 Subject: [PATCH 045/208] Fix import linting error --- lib/galaxy/webapps/galaxy/api/datatypes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/datatypes.py b/lib/galaxy/webapps/galaxy/api/datatypes.py index 89f0880b8dc..fe896cdb1f8 100644 --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -29,7 +29,6 @@ from galaxy.datatypes.schema import ( from galaxy.util import asbool from galaxy.web import expose_api_anonymous_and_sessionless from galaxy.webapps.base.controller import BaseAPIController - from . import get_app log = logging.getLogger(__name__) From ba1b26efb62573c245e06857675bb0e283d487ab Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 11 Jan 2021 13:43:14 -0500 Subject: [PATCH 046/208] Remove id field from tour yaml files Field not used. See code review discussion here: https://github.com/galaxyproject/galaxy/pull/11089#discussion_r555072412 --- config/plugins/tours/core.galaxy_ui.yaml | 1 - config/plugins/tours/core.history.yaml | 1 - config/plugins/tours/core.scratchbook.yaml | 1 - 3 files changed, 3 deletions(-) diff --git a/config/plugins/tours/core.galaxy_ui.yaml b/config/plugins/tours/core.galaxy_ui.yaml index e981beb72fe..dfc9e56f418 100644 --- a/config/plugins/tours/core.galaxy_ui.yaml +++ b/config/plugins/tours/core.galaxy_ui.yaml @@ -1,4 +1,3 @@ -id: galaxy_ui name: Galaxy UI description: A gentle introduction to the Galaxy User Interface title_default: "Welcome to Galaxy" diff --git a/config/plugins/tours/core.history.yaml b/config/plugins/tours/core.history.yaml index f0ce1e3b28a..7264e1538f3 100644 --- a/config/plugins/tours/core.history.yaml +++ b/config/plugins/tours/core.history.yaml @@ -1,4 +1,3 @@ -id: galaxy_history name: History Introduction description: A detailed introduction to the Galaxy History title_default: "Galaxy History Introduction" diff --git a/config/plugins/tours/core.scratchbook.yaml b/config/plugins/tours/core.scratchbook.yaml index 74e892eea62..b3111a98088 100644 --- a/config/plugins/tours/core.scratchbook.yaml +++ b/config/plugins/tours/core.scratchbook.yaml @@ -1,4 +1,3 @@ -id: galaxy_scratchbook name: Scratchbook - Introduction description: "An introduction on how to display multiple datasets and visualizations next to each other." title_default: "Scratchbook Introduction" From 75ed30543d7830e5c5f03d3b174ed114c146d14e Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 11 Jan 2021 15:22:04 -0500 Subject: [PATCH 047/208] Add model field metadata; revise tour model Address comments in code review Model is revised due to change in Tour model spec: - Id field is no longer part of the yaml file spec; - However, an id is generated and added to the tour data returned as a list - As a result, in a list, a tour item must have an id, whereas in tour details, a tour item must not have an id. I've factored out "core tour" data into its own model to solve this. --- lib/galaxy/tours/__init__.py | 7 ++- lib/galaxy/tours/schema.py | 85 ++++++++++++++++++++++++++++-------- 2 files changed, 72 insertions(+), 20 deletions(-) diff --git a/lib/galaxy/tours/__init__.py b/lib/galaxy/tours/__init__.py index cb77527700e..a59537d794b 100644 --- a/lib/galaxy/tours/__init__.py +++ b/lib/galaxy/tours/__init__.py @@ -8,7 +8,10 @@ import yaml from pydantic import parse_obj_as from galaxy import util -from galaxy.tours.schema import TourList +from galaxy.tours.schema import ( + TourDetails, + TourList, +) log = logging.getLogger(__name__) @@ -67,7 +70,7 @@ class ToursRegistry: if self._is_yaml(filename): self._load_tour_from_path(path) - def tour_contents(self, tour_id): + def tour_contents(self, tour_id) -> TourDetails: # Extra format translation could happen here (like the previous intro_to_tour) # For now just return the loaded contents. return self.tours.get(tour_id, None) diff --git a/lib/galaxy/tours/schema.py b/lib/galaxy/tours/schema.py index b738d762a5d..bf5d1274fef 100644 --- a/lib/galaxy/tours/schema.py +++ b/lib/galaxy/tours/schema.py @@ -3,31 +3,80 @@ from typing import ( Optional, ) -from pydantic import BaseModel +from pydantic import BaseModel, Field -class Tour(BaseModel): - id: str - name: str - description: str - tags: List[str] +class TourCore(BaseModel): + name: str = Field( + title='Name', + description='Name of tour' + ) + description: str = Field( + title='Description', + description='Tour description' + ) + tags: List[str] = Field( + title='Tags', + description='Topic topic tags' + ) + + +class Tour(TourCore): + id: str = Field( + title='Identifier', + description='Tour identifier' + ) class TourList(BaseModel): - __root__: List[Tour] = [] + __root__: List[Tour] = Field( + title='List of tours', + default=[] + ) class TourStep(BaseModel): - title: Optional[str] = None - content: Optional[str] = None - element: Optional[str] = None - placement: Optional[str] = None - preclick: Optional[list] = None - postclick: Optional[list] = None - textinsert: Optional[str] = None - backdrop: Optional[bool] = None + title: Optional[str] = Field( + title='Title', + description='Title displayed in the header of the step container' + ) + content: Optional[str] = Field( + title='Content', + description='Text shown to the user' + ) + element: Optional[str] = Field( + title='Element', + description='JQuery selector for the element to be described/clicked' + ) + placement: Optional[str] = Field( + title='Placement', + description='Placement of the text box relative to the selected element' + ) + preclick: Optional[list] = Field( + title='Pre-click', + description='Elements that receive a click() event before the step is shown' + ) + postclick: Optional[list] = Field( + title='Post-click', + description='Elements that receive a click() event after the step is shown' + ) + textinsert: Optional[str] = Field( + title='Text-insert', + description='Text to insert if element is a text box (e.g. tool search or upload)' + ) + backdrop: Optional[bool] = Field( + title='Backdrop', + description=('Show a dark backdrop behind the popover and its element,' + 'highlighting the current step') + ) -class TourDetails(Tour): - title_default: Optional[str] = None - steps: List[TourStep] +class TourDetails(TourCore): + title_default: Optional[str] = Field( + title='Default title', + description='Default title for each step' + ) + steps: List[TourStep] = Field( + title='Steps', + description='Tour steps' + ) From 8af7313f3d60d4a204c8d46fa92c47b64933af00 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 21 Oct 2020 23:10:54 -0400 Subject: [PATCH 048/208] Add a simpler 'get or create' api key endpoint --- lib/galaxy/webapps/galaxy/api/users.py | 14 +++++++++++++- lib/galaxy/webapps/galaxy/buildapp.py | 6 ++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py index c7184b8f00a..c935cc52c44 100644 --- a/lib/galaxy/webapps/galaxy/api/users.py +++ b/lib/galaxy/webapps/galaxy/api/users.py @@ -20,7 +20,10 @@ from galaxy import ( web ) from galaxy.exceptions import ObjectInvalid -from galaxy.managers import users +from galaxy.managers import ( + api_keys, + users +) from galaxy.security.validate_user_input import ( validate_email, validate_password, @@ -55,6 +58,7 @@ class UserAPIController(BaseAPIController, UsesTagsMixin, CreatesApiKeysMixin, B self.user_manager = users.UserManager(app) self.user_serializer = users.UserSerializer(app) self.user_deserializer = users.UserDeserializer(app) + self.api_key_manager = api_keys.ApiKeyManager(app) @expose_api def index(self, trans, deleted='False', f_email=None, f_name=None, f_any=None, **kwd): @@ -733,6 +737,14 @@ class UserAPIController(BaseAPIController, UsesTagsMixin, CreatesApiKeysMixin, B user = self._get_user(trans, id) return self.create_api_key(trans, user) + @expose_api + def get_or_create_api_key(self, trans, id, payload={}, **kwd): + """ + Unified 'get or create' for API key + """ + user = self._get_user(trans, id) + return self.api_key_manager.get_or_create_api_key(user) + @expose_api def get_api_key(self, trans, id, payload=None, **kwd): """ diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index f94878736e6..84060d7ff80 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -752,6 +752,12 @@ def populate_api_routes(webapp, app): action='api_key', conditions=dict(method=["POST"])) + webapp.mapper.connect('api_key', + '/api/users/{id}/api_key', + controller='users', + action='get_or_create_api_key', + conditions=dict(method=["GET"])) + webapp.mapper.connect('get_api_key', '/api/users/{id}/api_key/inputs', controller='users', From 16d0d74d09057e751ef50759db3b9a894c804e52 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 21 Oct 2020 23:15:37 -0400 Subject: [PATCH 049/208] Refactor dropping use of CreatesApiKeyMixin for direct manager use --- lib/galaxy/webapps/galaxy/api/users.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py index c935cc52c44..fda38cc0019 100644 --- a/lib/galaxy/webapps/galaxy/api/users.py +++ b/lib/galaxy/webapps/galaxy/api/users.py @@ -42,7 +42,6 @@ from galaxy.web.form_builder import AddressField from galaxy.webapps.base.controller import ( BaseAPIController, BaseUIController, - CreatesApiKeysMixin, UsesFormDefinitionsMixin, UsesTagsMixin ) @@ -51,7 +50,7 @@ from galaxy.webapps.base.controller import ( log = logging.getLogger(__name__) -class UserAPIController(BaseAPIController, UsesTagsMixin, CreatesApiKeysMixin, BaseUIController, UsesFormDefinitionsMixin): +class UserAPIController(BaseAPIController, UsesTagsMixin, BaseUIController, UsesFormDefinitionsMixin): def __init__(self, app): super().__init__(app) @@ -735,7 +734,7 @@ class UserAPIController(BaseAPIController, UsesTagsMixin, CreatesApiKeysMixin, B """ payload = payload or {} user = self._get_user(trans, id) - return self.create_api_key(trans, user) + return self.api_key_manager.create_api_key(trans, user) @expose_api def get_or_create_api_key(self, trans, id, payload={}, **kwd): @@ -761,7 +760,7 @@ class UserAPIController(BaseAPIController, UsesTagsMixin, CreatesApiKeysMixin, B """ payload = payload or {} user = self._get_user(trans, id) - self.create_api_key(trans, user) + self.api_key_manager.create_api_key(trans, user) return self._build_inputs_api_key(user, message='Generated a new web API key.') def _build_inputs_api_key(self, user, message=''): From d1be93182e8ee4e8def8fc33aa3420fed22cfd89 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Mon, 11 Jan 2021 15:37:31 -0500 Subject: [PATCH 050/208] sync optional/mutable props w/ rebased standard --- lib/galaxy/webapps/galaxy/api/users.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py index fda38cc0019..82c1227cc80 100644 --- a/lib/galaxy/webapps/galaxy/api/users.py +++ b/lib/galaxy/webapps/galaxy/api/users.py @@ -737,10 +737,11 @@ class UserAPIController(BaseAPIController, UsesTagsMixin, BaseUIController, Uses return self.api_key_manager.create_api_key(trans, user) @expose_api - def get_or_create_api_key(self, trans, id, payload={}, **kwd): + def get_or_create_api_key(self, trans, id, payload=None, **kwd): """ Unified 'get or create' for API key """ + payload = payload or {} user = self._get_user(trans, id) return self.api_key_manager.get_or_create_api_key(user) From 4f083e8c864aa1ab8cdb03634912df432a7b4e2e Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Mon, 11 Jan 2021 17:04:30 -0500 Subject: [PATCH 051/208] Fix extra arg left from the mixin refactoring. --- lib/galaxy/webapps/galaxy/api/users.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py index 82c1227cc80..b2e20d592a6 100644 --- a/lib/galaxy/webapps/galaxy/api/users.py +++ b/lib/galaxy/webapps/galaxy/api/users.py @@ -734,7 +734,7 @@ class UserAPIController(BaseAPIController, UsesTagsMixin, BaseUIController, Uses """ payload = payload or {} user = self._get_user(trans, id) - return self.api_key_manager.create_api_key(trans, user) + return self.api_key_manager.create_api_key(user) @expose_api def get_or_create_api_key(self, trans, id, payload=None, **kwd): From f1274c22caf73ccfe0bdb3cb2d4312862e887033 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 11 Jan 2021 17:42:58 -0500 Subject: [PATCH 052/208] Use a registry abc instead of concrete class --- lib/galaxy/tours/__init__.py | 11 +++++------ lib/galaxy/tours/abc/__init__.py | 21 +++++++++++++++++++++ lib/galaxy/webapps/galaxy/api/__init__.py | 5 +++++ lib/galaxy/webapps/galaxy/api/tours.py | 12 ++++++------ 4 files changed, 37 insertions(+), 12 deletions(-) create mode 100644 lib/galaxy/tours/abc/__init__.py diff --git a/lib/galaxy/tours/__init__.py b/lib/galaxy/tours/__init__.py index a59537d794b..478c0fb05ce 100644 --- a/lib/galaxy/tours/__init__.py +++ b/lib/galaxy/tours/__init__.py @@ -8,10 +8,8 @@ import yaml from pydantic import parse_obj_as from galaxy import util -from galaxy.tours.schema import ( - TourDetails, - TourList, -) +from galaxy.tours import abc +from galaxy.tours.schema import TourList log = logging.getLogger(__name__) @@ -31,12 +29,13 @@ def load_steps(contents_dict): step['title'] = title_default +@abc.ToursRegistry.register class ToursRegistry: def __init__(self, tour_directories): self.tour_directories = util.config_directories_from_setting(tour_directories) self.load_tours() - def tours_by_id_with_description(self) -> TourList: + def tours_by_id_with_description(self): tours = [] for k in self.tours.keys(): tourdata = { @@ -70,7 +69,7 @@ class ToursRegistry: if self._is_yaml(filename): self._load_tour_from_path(path) - def tour_contents(self, tour_id) -> TourDetails: + def tour_contents(self, tour_id): # Extra format translation could happen here (like the previous intro_to_tour) # For now just return the loaded contents. return self.tours.get(tour_id, None) diff --git a/lib/galaxy/tours/abc/__init__.py b/lib/galaxy/tours/abc/__init__.py new file mode 100644 index 00000000000..ba0b1f06c5d --- /dev/null +++ b/lib/galaxy/tours/abc/__init__.py @@ -0,0 +1,21 @@ +from abc import ABC, abstractmethod + +from galaxy.tours.schema import ( + TourDetails, + TourList, +) + + +class ToursRegistry(ABC): + + @abstractmethod + def tours_by_id_with_description(self) -> TourList: + """Return list of tours.""" + + @abstractmethod + def tour_contents(self, tour_id: str) -> TourDetails: + """Return tour details.""" + + @abstractmethod + def load_tour(self, tour_id: str) -> TourDetails: + """Reload tour and return tour details.""" diff --git a/lib/galaxy/webapps/galaxy/api/__init__.py b/lib/galaxy/webapps/galaxy/api/__init__.py index 4665a075b70..9f0dd36b6d7 100644 --- a/lib/galaxy/webapps/galaxy/api/__init__.py +++ b/lib/galaxy/webapps/galaxy/api/__init__.py @@ -24,6 +24,7 @@ from galaxy.managers.jobs import JobManager from galaxy.managers.session import GalaxySessionManager from galaxy.managers.users import UserManager from galaxy.model import User +from galaxy.tours.abc import ToursRegistry from galaxy.web.framework.decorators import require_admin_message from galaxy.work.context import SessionRequestContext @@ -85,3 +86,7 @@ def get_admin_user(trans: SessionRequestContext = Depends(get_trans)): if not trans.user_is_admin: raise AdminRequiredException(require_admin_message(trans.app.config, trans.user)) return trans.user + + +def get_tours_registry(app: UniverseApplication = Depends(get_app)) -> ToursRegistry: + return app.tour_registry diff --git a/lib/galaxy/webapps/galaxy/api/tours.py b/lib/galaxy/webapps/galaxy/api/tours.py index f8e9f97228d..dddcca3a364 100644 --- a/lib/galaxy/webapps/galaxy/api/tours.py +++ b/lib/galaxy/webapps/galaxy/api/tours.py @@ -7,7 +7,7 @@ from fastapi import Depends from fastapi_utils.cbv import cbv from fastapi_utils.inferring_router import InferringRouter as APIRouter -from galaxy.app import UniverseApplication +from galaxy.tours.abc import ToursRegistry from galaxy.tours.schema import ( TourDetails, TourList, @@ -20,7 +20,7 @@ from galaxy.web import ( from galaxy.webapps.base.controller import BaseAPIController from . import ( get_admin_user, - get_app, + get_tours_registry, ) log = logging.getLogger(__name__) @@ -31,22 +31,22 @@ router = APIRouter(tags=['tours']) @cbv(router) class FastAPITours: - app: UniverseApplication = Depends(get_app) + registry: ToursRegistry = Depends(get_tours_registry) @router.get('/api/tours') def index(self) -> TourList: """Return list of available tours.""" - return self.app.tour_registry.tours_by_id_with_description() + return self.registry.tours_by_id_with_description() @router.get('/api/tours/{tour_id}') def show(self, tour_id: str) -> TourDetails: """Return a tour definition.""" - return self.app.tour_registry.tour_contents(tour_id) + return self.registry.tour_contents(tour_id) @router.post('/api/tours/{tour_id}', dependencies=[Depends(get_admin_user)]) def update_tour(self, tour_id: str) -> TourDetails: """Return a tour definition.""" - return self.app.tour_registry.load_tour(tour_id) + return self.registry.load_tour(tour_id) class ToursController(BaseAPIController): From b9399bf051641ccb3a4373f1bbaf51b5152d8512 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 12 Jan 2021 10:49:45 +0100 Subject: [PATCH 053/208] Refactor move datatypes/schemas.py to managers/datatypes.py --- lib/galaxy/{datatypes/schema.py => managers/datatypes.py} | 0 lib/galaxy/webapps/galaxy/api/datatypes.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename lib/galaxy/{datatypes/schema.py => managers/datatypes.py} (100%) diff --git a/lib/galaxy/datatypes/schema.py b/lib/galaxy/managers/datatypes.py similarity index 100% rename from lib/galaxy/datatypes/schema.py rename to lib/galaxy/managers/datatypes.py diff --git a/lib/galaxy/webapps/galaxy/api/datatypes.py b/lib/galaxy/webapps/galaxy/api/datatypes.py index fe896cdb1f8..cb422ad606f 100644 --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -20,7 +20,7 @@ from fastapi_utils.cbv import cbv from galaxy.app import UniverseApplication from galaxy.datatypes.data import Data from galaxy.datatypes.registry import Registry -from galaxy.datatypes.schema import ( +from galaxy.managers.datatypes import ( DatatypeConverter, DatatypeDetails, DatatypesCombinedMap, From e87245b3255613d92dcfbbbf2b97878c141792f4 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 9 Dec 2020 17:45:30 +0100 Subject: [PATCH 054/208] SelectParameters: allow to choose legal options by name this change will allow choose a legal option by name. use case: data table with 2 columns: name and value (e.g. mothur data tables and probably many other old ones). tools using such data tables can not be tested since one needs to specify the value .. which is a path that changes with the test environment. this has been tested for one mothur tool --- lib/galaxy/tools/parameters/basic.py | 34 ++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index c756bba7a1f..17250c1b297 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -878,6 +878,9 @@ class SelectToolParameter(ToolParameter): return self.static_options def get_legal_values(self, trans, other_values): + """ + determine the set of values of legal options + """ if self.options: return {v for _, v, _ in self.options.get_options(trans, other_values)} elif self.dynamic_options: @@ -890,12 +893,23 @@ class SelectToolParameter(ToolParameter): else: return self.legal_values + def get_legal_names(self, trans, other_values): + """ + determine a mapping from names to values for all legal options + """ + return {n: v for n, v, _ in self.get_options(trans, other_values)} + def from_json(self, value, trans, other_values=None, require_legal_value=True): other_values = other_values or {} try: legal_values = self.get_legal_values(trans, other_values) except ImplicitConversionRequired: return value + # if the given value is not found in the set of values of the legal + # options we fall back to check if the value is in the set of names of + # the legal options. this is done with the fallback_values dict which + # allows to determine the corresponding legal values + fallback_values = self.get_legal_names(trans, other_values) if (not legal_values or not require_legal_value) and is_runtime_context(trans, other_values): if self.multiple: # While it is generally allowed that a select value can be '', @@ -926,9 +940,14 @@ class SelectToolParameter(ToolParameter): raise ParameterValueError("multiple values provided but parameter is not expecting multiple values", self.name, is_dynamic=self.is_dynamic) rval = [] for v in value: - if v not in legal_values: - raise ParameterValueError("an invalid option ({!r}) was selected (valid options: {})".format(v, ",".join(legal_values)), self.name, v, is_dynamic=self.is_dynamic) - rval.append(v) + if v in legal_values: + rval.append(v) + if len(rval) == 0: + for v in value: + if v in fallback_values: + rval.append(fallback_values[v]) + if len(rval) == 0: + raise ParameterValueError("an invalid option ({!r}) was selected (valid options: {})".format(v, ",".join(legal_values)), self.name, v, is_dynamic=self.is_dynamic) return rval else: value_is_none = (value == "None" and "None" not in legal_values) @@ -940,9 +959,14 @@ class SelectToolParameter(ToolParameter): raise ParameterValueError("no option was selected for non optional parameter", self.name, is_dynamic=self.is_dynamic) if is_runtime_value(value): return None - if value not in legal_values and require_legal_value: + if value in legal_values: + return value + elif value in fallback_values: + return fallback_values[value] + elif not require_legal_value: + return value + else: raise ParameterValueError("an invalid option ({!r}) was selected (valid options: {})".format(value, ",".join(legal_values)), self.name, value, is_dynamic=self.is_dynamic) - return value def to_param_dict_string(self, value, other_values=None): if value in (None, []): From d512049a4aa98c0cc82e5376703e6eebd7734697 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 9 Dec 2020 21:52:43 +0100 Subject: [PATCH 055/208] more specific functional tool tests involving fasta_indexes.loc rename entries such that they are suffixed by the column name (execpt for dbkey). this makes tests more specific, because this ensures that always the correct column is used. --- lib/galaxy_test/api/test_tools.py | 4 +-- test/functional/tool-data/fasta_indexes.loc | 4 +-- .../tools/dbkey_filter_collection.xml | 16 +++++----- test/functional/tools/dbkey_filter_input.xml | 6 ++-- .../tools/dbkey_filter_multi_input.xml | 6 ++-- test/functional/tools/dbkey_output_action.xml | 2 +- .../functional/tools/filter_static_regexp.xml | 32 +++++++++---------- test/functional/tools/remove_value.xml | 8 ++--- test/functional/tools/select_from_dataset.xml | 20 ++++++------ 9 files changed, 49 insertions(+), 49 deletions(-) diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py index e8c5d8f9e00..6a81373ebb4 100644 --- a/lib/galaxy_test/api/test_tools.py +++ b/lib/galaxy_test/api/test_tools.py @@ -1217,7 +1217,7 @@ class ToolsTestCase(ApiTestCase, TestsTools): tool_test_dicts = [{ "inputs": { "input": ["simple_line.txt"], - "index": ["hg18"], + "index": ["hg18_value"], }, "outputs": {}, "required_files": [['simple_line.txt', {'value': 'simple_line.txt', 'dbkey': 'hg18'}]], @@ -1235,7 +1235,7 @@ class ToolsTestCase(ApiTestCase, TestsTools): tool_test_dicts = [{ "inputs": { "input": ["simple_line.txt"], - "index": ["hg18"], + "index": ["hg18_value"], }, "outputs": {}, "required_files": [['simple_line.txt', {'value': 'simple_line.txt', 'dbkey': 'hgnot18'}]], diff --git a/test/functional/tool-data/fasta_indexes.loc b/test/functional/tool-data/fasta_indexes.loc index 3383ed89752..57f11daa9a9 100644 --- a/test/functional/tool-data/fasta_indexes.loc +++ b/test/functional/tool-data/fasta_indexes.loc @@ -1,2 +1,2 @@ -hg19 hg19 hg19 hg19 -hg18 hg18 hg18 hg18 \ No newline at end of file +hg19_value hg19 hg19_name hg19_path +hg18_value hg18 hg18_name hg18_path diff --git a/test/functional/tools/dbkey_filter_collection.xml b/test/functional/tools/dbkey_filter_collection.xml index eeb5937336e..ed8b5a13d8f 100644 --- a/test/functional/tools/dbkey_filter_collection.xml +++ b/test/functional/tools/dbkey_filter_collection.xml @@ -31,9 +31,9 @@ - + - + @@ -44,10 +44,10 @@ - + - + @@ -58,9 +58,9 @@ - + - + @@ -71,9 +71,9 @@ - + - + diff --git a/test/functional/tools/dbkey_filter_input.xml b/test/functional/tools/dbkey_filter_input.xml index ae04f6fc58f..8bf19cdd574 100644 --- a/test/functional/tools/dbkey_filter_input.xml +++ b/test/functional/tools/dbkey_filter_input.xml @@ -21,19 +21,19 @@ cat '$inputs' > '$output' - + - + - + diff --git a/test/functional/tools/dbkey_filter_multi_input.xml b/test/functional/tools/dbkey_filter_multi_input.xml index df04926938d..9463440c2af 100644 --- a/test/functional/tools/dbkey_filter_multi_input.xml +++ b/test/functional/tools/dbkey_filter_multi_input.xml @@ -23,19 +23,19 @@ - + - + - + diff --git a/test/functional/tools/dbkey_output_action.xml b/test/functional/tools/dbkey_output_action.xml index 128e43d721f..1d29a0b4e65 100644 --- a/test/functional/tools/dbkey_output_action.xml +++ b/test/functional/tools/dbkey_output_action.xml @@ -26,7 +26,7 @@ echo foo > '$mapped_reads' - + diff --git a/test/functional/tools/filter_static_regexp.xml b/test/functional/tools/filter_static_regexp.xml index ebfcd0756e8..090f8835e8d 100644 --- a/test/functional/tools/filter_static_regexp.xml +++ b/test/functional/tools/filter_static_regexp.xml @@ -45,10 +45,10 @@ echo $index_regexp_keep >> '$output' && - - - - + + + + @@ -60,24 +60,24 @@ echo $index_regexp_keep >> '$output' && - - - - + + + + - - - - + + + + - - - - + + + + diff --git a/test/functional/tools/remove_value.xml b/test/functional/tools/remove_value.xml index 5671e27e278..b01e01a24f5 100644 --- a/test/functional/tools/remove_value.xml +++ b/test/functional/tools/remove_value.xml @@ -6,8 +6,8 @@ echo $choose_value > '$out_file' - - + + @@ -21,12 +21,12 @@ echo $choose_value > '$out_file' - + - + diff --git a/test/functional/tools/select_from_dataset.xml b/test/functional/tools/select_from_dataset.xml index 3a67ac01db8..14c89aa1877 100644 --- a/test/functional/tools/select_from_dataset.xml +++ b/test/functional/tools/select_from_dataset.xml @@ -5,7 +5,7 @@ echo select_single '$select_single' > '$output' && echo select_multiple '$select_multiple' >> '$output' && echo select_collection '$select_collection' >> '$output' ]]> - + @@ -22,14 +22,14 @@ echo select_collection '$select_collection' >> '$output' - + - + @@ -39,16 +39,16 @@ echo select_collection '$select_collection' >> '$output' - + - - - - + + + + - - + + From 365e5a75c2381bf88a2248e0db8c70608f1ab6fa Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 10 Dec 2020 10:21:50 +0100 Subject: [PATCH 056/208] make 2nd column of 2.tabular test data unique values in preparation of a test for chosing selects per name --- test-data/2.tabular | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-data/2.tabular b/test-data/2.tabular index c1e73a28a26..52834cc059f 100644 --- a/test-data/2.tabular +++ b/test-data/2.tabular @@ -5,6 +5,6 @@ 5 58 3.2 6 60 3.1 7 67 3.8 -8 68 4.1 -9 71 4.3 +8 168 4.1 +9 171 4.3 10 69 3.7 From 0c8c403bf42c6ae2d91692af4f8b61f50bd1f2df Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 10 Dec 2020 11:34:25 +0100 Subject: [PATCH 057/208] add tests showing that dynamic selects can be chosen by name for from_data_table and from_dataset --- test/functional/tools/dbkey_filter_input.xml | 18 ++++++++++++++++- test/functional/tools/select_from_dataset.xml | 20 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/test/functional/tools/dbkey_filter_input.xml b/test/functional/tools/dbkey_filter_input.xml index 8bf19cdd574..15daef6f8b2 100644 --- a/test/functional/tools/dbkey_filter_input.xml +++ b/test/functional/tools/dbkey_filter_input.xml @@ -1,7 +1,8 @@ Filter (single) input on a dbkey '$output' + cat '$inputs' > '$output' && + echo $index ]]> @@ -23,12 +24,18 @@ cat '$inputs' > '$output' + + + + + + @@ -36,6 +43,15 @@ cat '$inputs' > '$output' + + + + + + + + + diff --git a/test/functional/tools/select_from_dataset.xml b/test/functional/tools/select_from_dataset.xml index 14c89aa1877..1cb117129ef 100644 --- a/test/functional/tools/select_from_dataset.xml +++ b/test/functional/tools/select_from_dataset.xml @@ -57,6 +57,26 @@ echo select_collection '$select_collection' >> '$output' + + + + + + + + + + + + + + + + + + + + From 1fb1bb91c41227aa3f9fa47b55b5c32e7e407de2 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 10 Dec 2020 14:39:12 +0100 Subject: [PATCH 058/208] document test parameters for the simple cases that deserve docs --- lib/galaxy/tool_util/xsd/galaxy.xsd | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 3a5dd199908..a449fab1693 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -1354,6 +1354,38 @@ This tag set defines the tool's input parameters for executing the tool via the functional test framework. See [test](#tool-tests-test) documentation for some simple examples of parameters. +### Parameter Types + +#### ``text``, ``integer``, and ``float`` + +Values for these parameters are simply given by the desired value. + +#### ``boolean`` + +The value of the test parameter should be set to `true` or `false` corresponding +to the cases that the parameter is checked or not. Alternatively the value +specified as `truevalue` or `falsevalue` can be given. + +#### ``data`` + +Data input parameters can be given as a file name. The file should exist in the +`test-data` folder. Multiple files can be specified as comma separated list. + +#### ``select`` + +The value of a select parameter should be specified as the value of one of the +legal options. If more than one option is selected (`multiple="true"`) they +should be given as comma separated list. For optional selects +(`optional="true"`) the case that no option is selected can be specified with +`value=""`. + +While in general it is preferred to specify the selected cases by their values it +is also possible to specify them by their name (i.e. the content of the +`option` tag that is shown to the user). One use case is a dynamic select that is +generated from a data table with two columns: name and value where the value is +a path. Since the path changes with the test environment it can not be used to +select an option for a test. + ]]> From 857265c24707ca5b88664c7de8747058e45a8a88 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 10 Dec 2020 14:41:35 +0100 Subject: [PATCH 059/208] code simplification --- lib/galaxy/tools/parameters/basic.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 17250c1b297..8d090ee158c 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -881,17 +881,7 @@ class SelectToolParameter(ToolParameter): """ determine the set of values of legal options """ - if self.options: - return {v for _, v, _ in self.options.get_options(trans, other_values)} - elif self.dynamic_options: - try: - call_other_values = self._get_dynamic_options_call_other_values(trans, other_values) - return {v for _, v, _ in eval(self.dynamic_options, self.tool.code_namespace, call_other_values)} - except Exception as e: - log.debug("Determining legal values failed for '%s': %s", self.name, unicodify(e)) - return set() - else: - return self.legal_values + return {v for _, v, _ in self.get_options(trans, other_values)} def get_legal_names(self, trans, other_values): """ From 56d5822862d70d0d1dd3219dd27b63e791524a46 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 12 Jan 2021 13:53:05 +0100 Subject: [PATCH 060/208] Add view functions --- lib/galaxy/managers/datatypes.py | 92 +++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/managers/datatypes.py b/lib/galaxy/managers/datatypes.py index 1e82c249f1a..1512e699419 100644 --- a/lib/galaxy/managers/datatypes.py +++ b/lib/galaxy/managers/datatypes.py @@ -1,4 +1,9 @@ -from typing import Dict, List, Optional +from typing import ( + Dict, + List, + Optional, + Union, +) from pydantic import ( BaseModel, @@ -6,6 +11,9 @@ from pydantic import ( HttpUrl ) +from galaxy.datatypes.data import Data +from galaxy.datatypes.registry import Registry + class CompositeFileInfo(BaseModel): name: str = Field( @@ -116,3 +124,85 @@ class DatatypeConverter(BaseModel): description="The converter tool identifier", example="CONVERTER_Bam_Bai_0", ) + + +def view_index( + datatypes_registry: Registry, + extension_only: Optional[bool] = True, + upload_only: Optional[bool] = True +) -> Union[List[DatatypeDetails], List[str]]: + if extension_only: + if upload_only: + return datatypes_registry.upload_file_formats + else: + return [ext for ext in datatypes_registry.datatypes_by_extension] + else: + rval = [] + for datatype_info_dict in datatypes_registry.datatype_info_dicts: + if upload_only and not datatype_info_dict.get('display_in_upload'): + continue + rval.append(datatype_info_dict) + return rval + + +def view_mapping(datatypes_registry: Registry) -> DatatypesMap: + ext_to_class_name: Dict[str, str] = {} + classes = [] + for k, v in datatypes_registry.datatypes_by_extension.items(): + c = v.__class__ + ext_to_class_name[k] = f"{c.__module__}.{c.__name__}" + classes.append(c) + class_to_classes: Dict[str, Dict[str, bool]] = {} + + def visit_bases(types, cls): + for base in cls.__bases__: + if issubclass(base, Data): + types.add(f"{base.__module__}.{base.__name__}") + visit_bases(types, base) + + for c in classes: + n = f"{c.__module__}.{c.__name__}" + types = {n} + visit_bases(types, c) + class_to_classes[n] = {t: True for t in types} + return DatatypesMap(ext_to_class_name=ext_to_class_name, class_to_classes=class_to_classes) + + +def view_types_and_mapping( + datatypes_registry: Registry, + extension_only: Optional[bool] = True, + upload_only: Optional[bool] = True +) -> DatatypesCombinedMap: + return DatatypesCombinedMap( + datatypes=view_index(datatypes_registry, extension_only, upload_only), + datatypes_mapping=view_mapping(datatypes_registry) + ) + + +def view_sniffers(datatypes_registry: Registry) -> List[str]: + rval: List[str] = [] + for sniffer_elem in datatypes_registry.sniffer_elems: + datatype = sniffer_elem.get('type') + if datatype is not None: + rval.append(datatype) + return rval + + +def view_converters(datatypes_registry: Registry) -> List[DatatypeConverter]: + converters: List[DatatypeConverter] = [] + for (source_type, targets) in datatypes_registry.datatype_converters.items(): + for target_type in targets: + converters.append(DatatypeConverter( + source=source_type, + target=target_type, + tool_id=targets[target_type].id, + )) + return converters + + +def view_edam_formats(datatypes_registry: Registry) -> Dict[str, str]: + return datatypes_registry.edam_formats + + +def view_edam_data(datatypes_registry: Registry) -> Dict[str, str]: + return datatypes_registry.edam_data From 0e3715806eb604a811107a6cc3d2c23ae3b26800 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 12 Jan 2021 13:56:07 +0100 Subject: [PATCH 061/208] Refactor FastAPI and legacy controller to reuse views --- lib/galaxy/webapps/galaxy/api/datatypes.py | 241 ++++++--------------- 1 file changed, 62 insertions(+), 179 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/datatypes.py b/lib/galaxy/webapps/galaxy/api/datatypes.py index cb422ad606f..820aadff2c7 100644 --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -16,15 +16,18 @@ from fastapi import ( from fastapi.routing import APIRouter from fastapi_utils.cbv import cbv - from galaxy.app import UniverseApplication -from galaxy.datatypes.data import Data from galaxy.datatypes.registry import Registry from galaxy.managers.datatypes import ( DatatypeConverter, DatatypeDetails, DatatypesCombinedMap, DatatypesMap, + view_converters, view_edam_data, view_edam_formats, + view_index, + view_mapping, + view_sniffers, + view_types_and_mapping ) from galaxy.util import asbool from galaxy.web import expose_api_anonymous_and_sessionless @@ -56,145 +59,88 @@ def get_datatypes_registry(app: UniverseApplication = Depends(get_app)) -> Regis class FastAPIDatatypes: datatypes_registry: Registry = Depends(get_datatypes_registry) - @router.get('/api/datatypes', + @router.get( + '/api/datatypes', summary="Lists all available data types", response_model=Union[List[DatatypeDetails], List[str]], - response_description="List of data types") - async def index(self, - extension_only: Optional[bool] = ExtensionOnlyQueryParam, - upload_only: Optional[bool] = UploadOnlyQueryParam - ) -> Union[List[DatatypeDetails], List[str]]: + response_description="List of data types", + ) + async def index( + self, + extension_only: Optional[bool] = ExtensionOnlyQueryParam, + upload_only: Optional[bool] = UploadOnlyQueryParam + ) -> Union[List[DatatypeDetails], List[str]]: """Gets the list of all available data types.""" - try: - return self._index(extension_only, upload_only) - except Exception as e: - log.exception('Could not get datat ypes') - raise e + return view_index(self.datatypes_registry, extension_only, upload_only) - @router.get('/api/datatypes/mapping', + @router.get( + '/api/datatypes/mapping', summary="Returns mappings for data types and their implementing classes", response_model=DatatypesMap, - response_description="Dictionary to map data types with their classes") + response_description="Dictionary to map data types with their classes" + ) async def mapping(self) -> DatatypesMap: """Gets mappings for data types.""" - try: - return self._mapping() - except Exception as e: - log.exception('Could not get data type mapping') - raise e + return view_mapping(self.datatypes_registry) - @router.get('/api/datatypes/types_and_mapping', + @router.get( + '/api/datatypes/types_and_mapping', summary="Returns all the data types extensions and their mappings", response_model=DatatypesCombinedMap, - response_description="Dictionary to map data types with their classes") - async def types_and_mapping(self) -> DatatypesCombinedMap: + response_description="Dictionary to map data types with their classes" + ) + async def types_and_mapping( + self, + extension_only: Optional[bool] = ExtensionOnlyQueryParam, + upload_only: Optional[bool] = UploadOnlyQueryParam + ) -> DatatypesCombinedMap: """Combines the datatype information from (/api/datatypes) and the mapping information from (/api/datatypes/mapping) into a single response.""" - return DatatypesCombinedMap(datatypes=self._index(), datatypes_mapping=self._mapping()) + return DatatypesCombinedMap( + datatypes=view_index(self.datatypes_registry, extension_only, upload_only), + datatypes_mapping=view_mapping(self.datatypes_registry) + ) - @router.get('/api/datatypes/sniffers', + @router.get( + '/api/datatypes/sniffers', summary="Returns the list of all installed sniffers", response_model=List[str], - response_description="List of datatype sniffers") + response_description="List of datatype sniffers" + ) async def sniffers(self) -> List[str]: """Gets the list of all installed data type sniffers.""" - try: - return self._sniffers() - except Exception as e: - log.exception('Could not get sniffers') - raise e + return view_sniffers(self.datatypes_registry) - @router.get('/api/datatypes/converters', + @router.get( + '/api/datatypes/converters', summary="Returns the list of all installed converters", response_model=List[DatatypeConverter], - response_description="List of all datatype converters") + response_description="List of all datatype converters" + ) async def converters(self) -> List[DatatypeConverter]: """Gets the list of all installed converters.""" - try: - return self._converters() - except Exception as e: - log.exception('Could not get converters') - raise e + return view_converters(self.datatypes_registry) - @router.get('/api/datatypes/edam_formats', + @router.get( + '/api/datatypes/edam_formats', summary="Returns a dictionary/map of datatypes and EDAM formats", response_model=Dict[str, str], - response_description="Dictionary/map of datatypes and EDAM formats") + response_description="Dictionary/map of datatypes and EDAM formats" + ) async def edam_formats(self) -> Dict[str, str]: """Gets a map of datatypes and their corresponding EDAM formats.""" - try: - return self.datatypes_registry.edam_formats - except Exception as e: - log.exception('Could not get EDAM format mappings') - raise e + return self.datatypes_registry.edam_formats - @router.get('/api/datatypes/edam_data', + @router.get( + '/api/datatypes/edam_data', summary="Returns a dictionary/map of datatypes and EDAM data", response_model=Dict[str, str], - response_description="Dictionary/map of datatypes and EDAM data") + response_description="Dictionary/map of datatypes and EDAM data" + ) async def edam_data(self) -> Dict[str, str]: """Gets a map of datatypes and their corresponding EDAM data.""" - try: - return self.datatypes_registry.edam_data - except Exception as e: - log.exception('Could not get EDAM data mappings') - raise e - - def _index(self, extension_only: Optional[bool] = True, upload_only: Optional[bool] = True) -> Union[List[DatatypeDetails], List[str]]: - if extension_only: - if upload_only: - return self.datatypes_registry.upload_file_formats - else: - return [ext for ext in self.datatypes_registry.datatypes_by_extension] - else: - rval = [] - for datatype_info_dict in self.datatypes_registry.datatype_info_dicts: - if upload_only and not datatype_info_dict.get('display_in_upload'): - continue - rval.append(datatype_info_dict) - return rval - - def _mapping(self) -> DatatypesMap: - ext_to_class_name: Dict[str, str] = {} - classes = [] - for k, v in self.datatypes_registry.datatypes_by_extension.items(): - c = v.__class__ - ext_to_class_name[k] = f"{c.__module__}.{c.__name__}" - classes.append(c) - class_to_classes: Dict[str, Dict[str, bool]] = {} - - def visit_bases(types, cls): - for base in cls.__bases__: - if issubclass(base, Data): - types.add(f"{base.__module__}.{base.__name__}") - visit_bases(types, base) - - for c in classes: - n = f"{c.__module__}.{c.__name__}" - types = {n} - visit_bases(types, c) - class_to_classes[n] = {t: True for t in types} - return DatatypesMap(ext_to_class_name=ext_to_class_name, class_to_classes=class_to_classes) - - def _sniffers(self) -> List[str]: - rval: List[str] = [] - for sniffer_elem in self.datatypes_registry.sniffer_elems: - datatype = sniffer_elem.get('type') - if datatype is not None: - rval.append(datatype) - return rval - - def _converters(self) -> List[DatatypeConverter]: - converters: List[DatatypeConverter] = [] - for (source_type, targets) in self.datatypes_registry.datatype_converters.items(): - for target_type in targets: - converters.append(DatatypeConverter( - source=source_type, - target=target_type, - tool_id=targets[target_type].id, - )) - return converters + return self.datatypes_registry.edam_data class DatatypesController(BaseAPIController): @@ -205,28 +151,9 @@ class DatatypesController(BaseAPIController): GET /api/datatypes Return an object containing upload datatypes. """ - try: - return self._index(trans, **kwd) - except Exception as e: - log.exception('Could not get datatypes') - raise e - - def _index(self, trans, **kwd): - datatypes_registry = self._datatypes_registry extension_only = asbool(kwd.get('extension_only', True)) upload_only = asbool(kwd.get('upload_only', True)) - if extension_only: - if upload_only: - return datatypes_registry.upload_file_formats - else: - return [ext for ext in datatypes_registry.datatypes_by_extension] - else: - rval = [] - for datatype_info_dict in datatypes_registry.datatype_info_dicts: - if not datatype_info_dict.get('display_in_upload') and upload_only: - continue - rval.append(datatype_info_dict) - return rval + return view_index(self._datatypes_registry, extension_only, upload_only) @expose_api_anonymous_and_sessionless def mapping(self, trans, **kwd): @@ -234,32 +161,7 @@ class DatatypesController(BaseAPIController): GET /api/datatypes/mapping Return a dictionary of class to class mappings. ''' - try: - return self._mapping() - except Exception as e: - log.exception('Could not get datatype mapping') - raise e - - def _mapping(self): - ext_to_class_name = dict() - classes = [] - for k, v in self._datatypes_registry.datatypes_by_extension.items(): - c = v.__class__ - ext_to_class_name[k] = c.__module__ + "." + c.__name__ - classes.append(c) - class_to_classes = dict() - - def visit_bases(types, cls): - for base in cls.__bases__: - if issubclass(base, Data): - types.add(base.__module__ + "." + base.__name__) - visit_bases(types, base) - for c in classes: - n = c.__module__ + "." + c.__name__ - types = {n} - visit_bases(types, c) - class_to_classes[n] = {t: True for t in types} - return dict(ext_to_class_name=ext_to_class_name, class_to_classes=class_to_classes) + return view_mapping(self._datatypes_registry) @expose_api_anonymous_and_sessionless def types_and_mapping(self, trans, **kwd): @@ -270,10 +172,9 @@ class DatatypesController(BaseAPIController): mapping information from (/api/datatypes/mapping) into a single response. """ - return { - "datatypes": self._index(trans, **kwd), - "datatypes_mapping": self._mapping(), - } + extension_only = asbool(kwd.get('extension_only', True)) + upload_only = asbool(kwd.get('upload_only', True)) + return view_types_and_mapping(self._datatypes_registry, extension_only, upload_only) @expose_api_anonymous_and_sessionless def sniffers(self, trans, **kwd): @@ -281,37 +182,19 @@ class DatatypesController(BaseAPIController): GET /api/datatypes/sniffers Return a list of sniffers. ''' - try: - rval = [] - for sniffer_elem in self._datatypes_registry.sniffer_elems: - datatype = sniffer_elem.get('type') - if datatype is not None: - rval.append(datatype) - return rval - except Exception as e: - log.exception('Could not get datatypes') - raise e + return view_sniffers(self._datatypes_registry) @expose_api_anonymous_and_sessionless def converters(self, trans, **kwd): - converters = [] - for (source_type, targets) in self._datatypes_registry.datatype_converters.items(): - for target_type in targets: - converters.append({ - 'source': source_type, - 'target': target_type, - 'tool_id': targets[target_type].id, - }) - - return converters + return view_converters(self._datatypes_registry) @expose_api_anonymous_and_sessionless def edam_formats(self, trans, **kwds): - return self._datatypes_registry.edam_formats + return view_edam_formats(self._datatypes_registry) @expose_api_anonymous_and_sessionless def edam_data(self, trans, **kwds): - return self._datatypes_registry.edam_data + return view_edam_data(self._datatypes_registry) @property def _datatypes_registry(self): From afc87f99e4bf504ca948448df9a2606a09237b6f Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 12 Jan 2021 08:48:21 -0500 Subject: [PATCH 062/208] Test deploy w/ prebuilt files --- config/plugins/visualizations/nora/package.json | 15 +++++++-------- .../plugins/visualizations/nora/static/logo.png | Bin 0 -> 5502 bytes 2 files changed, 7 insertions(+), 8 deletions(-) create mode 100644 config/plugins/visualizations/nora/static/logo.png diff --git a/config/plugins/visualizations/nora/package.json b/config/plugins/visualizations/nora/package.json index 8e8b9ed0cb5..04adc1bd2ff 100644 --- a/config/plugins/visualizations/nora/package.json +++ b/config/plugins/visualizations/nora/package.json @@ -1,19 +1,18 @@ { "name": "nora", + "license": "MIT", + "description": "NORA Medical Image Viewer", "version": "1.1.0", "keywords": [ "galaxy", "visualization" ], - "description": "nora", "main": "index.js", - "scripts": { - "build": "cpy --cwd='node_modules/@reisertm/nora_galaxy_plugin' --parents './' '../../../static/dist'" - }, - "license": "MIT", "dependencies": { - "cpy-cli": "^3.1.1", - "underscore": "^1.8.3", - "@reisertm/nora_galaxy_plugin": "^1.1.6" + "@galaxyproject/nora": "^1.1.0", + "cpy-cli": "^3.1.1" + }, + "scripts": { + "build": "cpy --cwd='node_modules/@galaxyproject/nora/static/dist' --parents './' '../../../../../static/dist'" } } diff --git a/config/plugins/visualizations/nora/static/logo.png b/config/plugins/visualizations/nora/static/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..987efb0a6d24be06ce96ba05ba5d4e8fa707363f GIT binary patch literal 5502 zcmb_g^;Z;J6J9!&5?8t#K|&;?L0~CCI>nX7pu4+!>6Ff8L2`j5q(mg7J49NfmtMYo z|A%kRx$(=LGxy$^XJ($5H(Fp)Vme{~06?mys-%lqyD%ez5F0Ze?Ukiq79wX=6E^^W ztpC4(mCi>-4*)z7Qd3gU_s%=acW~ANw+xvX&3+ik;1v9x+iAJr-AgZ)HGl$ODO&V& z@{1Dr{IQ|^9^P3k1W(;Chz;jYe~O>V#+uAJ{S$=sje~+Rm4pHX5gNpgKM56{XX$kM z%4u?oJPM1UXvvtpNbg8Tbdw9Xc}!m#-_5MF-sVj)#1;d9ahDX8jF1x};KmV`;F8(o^9iTWrH!yFn3fXl|xzQJ5CKoT0bw%uB^P?L`pMhb5JU)s60*oQCeEs z+1VK%Pa<15-UgKo4ZEB2B3IVof(}E3vNXA*-->2y-l7hr+;@{lZF3=d*dY+q--S48 zK>-1Q?Gx|#wjHVCbTVZfc6N5j{6r45=`Qc`=4Own^4A}lb5ds6*x3txwzIcJGUJrk zROrs}vefOYt(9tKo(s^$7VkTGiPrcvvzN=W%JFjX^QrZ2Jr~&!p&>DA`BYU~Yh5Y| zkT*7-Uu^!E$#3#)!#V2%RG}W4IBGj(|H8JBc5vR3(`k9n@sMF+KLIO4y{7qF{V zDDTmbR4H+>w?SH3%d$!@};o7I^!4T6TE|B4q!9j)r$h z)pc&;M+vo47XPfUpbqGb*GB=-%BjcfoR%AK>MHFn*(JFbznes3i*t@pKS$NJDYXXi zpaY9il~cPU!NuYJl~o@V6&3aM^=)i8_1H{2@;P{u6OG8shRo^&zbD+HCpA7gT ziUHwYy}iA_*BV&&p{}~ys%6oVy$q|M9DH(eV%czYx;2swl@7RZ_xAQSHKo)0 ztVO|JY9&Oo?lHCatNYSEVMpvWRpUTLb}0a0-sJ)>Y92u%r8$IT-v8J=GNixWO4$!8 zXj`oLC52+}EkRVlI@;Cp>8P0)cKuasz-77@DAa0Kz*0%gG3JgOT>r`Qd9KeBa4s{L zq4x>^p!Lo%gn{5`U2|fhy;$y!L)3*4eJpm+Lqqj%(;B{!RfUy|06Sj-52hvm>#fWu z#CU0N=VXrSSu-b5ad8ohDhiFL;>zA$1^71Z>(4UD8B{fgo_=xw`A1ZLf6dF98i+0< zrk&@I4P%g-r6dC;Bvp)(Ao83t4L*@Ix-Guy{R%!OvcykHp#iR$F@;%Gv^##``QY8Z zyYSBlzZoAOZ|YxK!;R9xy$jycD**?2$?<)4JFr-Tp=6V6@$m4nv$KUrzm*u)$lago z*)El0sqbOp)TvvBfh>x;C!&@5xD5cX09;mGzjU0P$rvAD2*^5`*_FJFWTl5A7s#v( zEiA|uSfmNa7%Jg#yG|3hJTD{6@L;t^C6zA!egD;j<{o+9pRq+JXFU}t3GvC zD7y`1M}iR_I(d+R%x>YUjajSXMX>{+z}t5Btsm)mc?`C>5d_)pDk|oIHm|7;GLRKZ zk_3N^7X6|;8EUV_zy8G&(cRhNC~R|gdvkD2-~GvPFpIB<(#1zDBezv4TwQ_z@baVA zbe^BOy1M7ls*HpLp#%AFSHLZWI>AD)jS!7F(!J^Hu#UKAmyF(=Naf~PTad9Vbo@+c z)5JI{C!JvnW>iF}(~0fb%47F|q?9vh$gIgerFG2v&oaWTudiN2WKe;f5jy5w zKx4zGilO)_9)Lq>_~HCk*C%?uiZ)XtEz>$Z(|YRcvxAFd{K9$K?z}3me`8r$+4-GI z*up1SpHq&_QN>Ud$$#R5qNbUQlga#_L~-ohJ9#QEmCm)G8LKYH5}4K|fYkSJOB>rhMVXcsWS~#c%gj%!tC3 zwePnCl7Y506AGaD6pT#OgqeiiKkr`hsW&QPuaZBx;MlXCh0<^a`CRiS(LdT89v^pd zaCrZ%mq6dhNMwmc+P-u%khWfeZzC7lya_+^S*D%(4FG`f67|KjI1<9q_GvQ9Ms!ay)YnTN7Na#R%JY!6xM&LHMx`ez?#RW)sx1;6<=o8_(Adfl} zs;3^^%)RsciJrK!fC)l^831XS*R+l6zjd@Fd72(OzVk|@p?7L_Hd0BFa9fhe>Ps8$ ziMr2;-kBT5QQL4?*9ok-!1qUuWg?TC06_kI-;m(TwTF-PhjotDT$1yTWRs*^Fc5-` zjqNYIvbTRq0a|CLIF}U4&ByKK9nt0;@uFOCN9+|$@qXu-N!Sl{kIZzWRjSe#d+|bO ziK`m+1{XPoYqb-3{;x$&R5DZBPJ)7IOp~ICz1AI)(!PR2p1z(N@SB~h;6a)EyvSlT zHJ4690?5Y1DMz~Yi`d(@>0Izwox$+ZzD}%l?T~IdF$=pbTw1ACS4FFk?irPiDj-k~ z(SI-Ny0EO-5PWH+945ma$|5mfLZporAQ=B7F7*lCCv*XOE7e`|f+k?p2+`Rx<| zgZwR9kVizI(=&sHF0W)Yek$&l;eo`z9iFS{K3o|!gID5p*A_2E47l9%xR|?nkN~3J z78UJkE2||Ttq%>w@5j!S{DwsEa}zp^gLYiqI@&)v_ghl|uY^-3m#4FeNy=(kHJ|HN zQM=G(g&9kVTKuCYIY@Z6-=$v$y*q9h3lZ#syK5#rHzT3T-?)K|6^@B_9y2S)=Pii8oUeus_7nwP|YNK$hvbYRHW$t z2uE&%x47Ph7)v6mRuAc;Bcu)PUb$H(j0yeq6kK_P{3Q)YiP-#DTvB2LGmTh`iwQNB zjG+AyJ;CCAJ0LPOByHPa8s-_{*nhB)F6jNYVTg*~ICwFR2&2$E)Z0@>)|&Nuzy0;q zx}a0vNZGuSO|3~Ie$CUjtD%8K{06iqcO_b1dxt3Kc>~Hiz5QQ4Q~NR@EHy9lep23n zLcr{;G?Pi_|q*?x>0KioEa1IJCl#ae6p zUMxP+OO9ryXU&%twT)dSM{5)p7tg}XqFyPOs0K<+(sJr@`IM?gECO@>Y;6s1xdvsQ z->Kl&u91XoT^>q-DjYHZ5oz&rzs<%ZRgY5x`2u6FO_=Cc~Qh<34ia4~=$AIoK-~$W)T6Q!945 zQ%#?HS>WnxOjqy*bg|;K2D3meGJD`m$mX>+HTk0!?2-pxNp9K^j3LN5J|FI`UC3C0 zoo+xiGwnWS+k#zJ-@5U6!4zIcLrKhXzUMsgTSbi?rqMjGcC-9uaVhB>Ry%w9-Mj0v zN@Nzrt{r}15ZcGgEZ2`7y0Cxpg;lD1`bHtL(p=zOyF2d(K`vb1J!Afi*STVd!t*8% zGUM8M4F&Nd)bowZab7{eos%zQj|U!~PhtUgi2sds;%+U5oVu`z@p za0G~+Hr4=C#iyEDTXo`(e%7+i#t4uHgNHWyqVl^ylpN{_VX*s`IGev)J!=gQ#`2|T z;kR_0Qlc_sPjg$WO8d*A+_--LeBuzo5^+#3JfX!=KmKC}UW( zYd-2+B7Cc8NAAzo*CBF0nq0{fCx=HzNeP2pNxziR(Q1CXKcVrd%C? zS?Y;IR^J&TPu5H|t$MU1v9Pd?)xB*%X#pV|&jZPUfdPvJZS9!AAP;YklJ-WY=y9NGLY%60 zR_Iw|_?z9Ks>sB3PDZ9A0=Zuek9q!#pGC>X>D=+uEX#P48b?G#9G5)t{~~3$+~Gyu z4Ez~v1lKb)CAJbP6+vH2Txa+Ne1)8xX#gt}2`Jd2IEEYT#*0y*v_ktHp`oF!Zf@`S z$qVW}BjyMIG&D>L7Xh4gd=gso-9d6lQz%WS4qQ)H7aKt-A#?{>^6~DsZ%&b|)}5G} z^V%FFg&ES{@wp=MR$c1{EEkY|mqo9lXuK$&(5TsAa#?bQh{)r=w?b4&=^W7_iF@e^%rm68(-1c{1@a%<%|Kk+=QzywN!VoDA%JyMiU zL{^{e!>5A+HYmvUYX42g;Zmyblk4 z)VRI0)OwQ9WeC#RUb=>JJYs$ZOj2TtiH<)0w1Y#T%FW9=>~hb}&24uY;2bgYLB{hh%?H6w7IATE zgWqh}d(MdEY!SP~hwSup(QDbZ;+B@#ZmNG2+jS)-l=aLl^iQ6or-kcu_7B8S|7R4g zLRxw{Yll>7$5^%qn1d1v@Z-midUX)yBH+VNY7iAwWE6WGojf~Zit2g)V{HmtELt1k zm-fwzY8HeAEZVZy9N#+Fm->7$fSsG0t0TiGQP$PPv;@jKDJ?G_E0Cct!ukH&!}|Hc zU;;gB2|pLX108I(?rofBEtqzn6FM9kCR|kU6imDie4PYq60k9A%azR=3lSXYj{I8!^cl4co0}4vtpHEdH+sLUf z$nK0VN0=w>wt5B9dt_Z^#4}-f|uZd{Re9dsh zjhl-D3}F>M;Yu&z&=j>D(f83_I)~C=@DpwI0^ruQZ()k0-bn&jupK z!w%7^m^!$C)^{$=&dx3@>>YGW$n$_?FPGhF3}N3kh}@+m)CmAR8_pb*Y7of!_wUiY z5N8idQg+DvRlJkK4)uxX#f%=oDWZ&_+RQ{GrhYdm;kVm25HOjipO8it9Vy(fL0R(z+?PV7p Date: Tue, 12 Jan 2021 08:49:17 -0500 Subject: [PATCH 063/208] Fix entrypoint --- config/plugins/visualizations/nora/config/nora.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/plugins/visualizations/nora/config/nora.xml b/config/plugins/visualizations/nora/config/nora.xml index 7e2bd26134a..f84da5b5056 100644 --- a/config/plugins/visualizations/nora/config/nora.xml +++ b/config/plugins/visualizations/nora/config/nora.xml @@ -19,7 +19,7 @@ dataset_id - + svg optional From 386470789f251193e8cec749c35487e12aaaead6 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 11 Jan 2021 16:58:08 -0500 Subject: [PATCH 064/208] A couple new ways to speed up doc building locally. --- Makefile | 3 +++ doc/source/conf.py | 8 +++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0815ade564c..2e123d225b2 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,9 @@ docs: ## Generate HTML documentation. $(IN_VENV) $(MAKE) -C doc clean $(IN_VENV) $(MAKE) -C doc html +rebuild-docs: ## Re-generate HTML documentation without clean step. + $(IN_VENV) $(MAKE) -C doc html + setup-venv: if [ ! -f $(VENV)/bin/activate ]; then bash scripts/common_startup.sh --dev-wheels; fi diff --git a/doc/source/conf.py b/doc/source/conf.py index d44ee73b861..b2add06a301 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -18,6 +18,10 @@ import sphinx_rtd_theme # Library to make .md to slideshow from recommonmark.transform import AutoStructify +# Set GALAXY_DOCS_SKIP_VIEW_CODE=1 to skip embedding highlighted source +# code into docs. +SKIP_VIEW_CODE = os.environ.get("GALAXY_DOCS_SKIP_VIEW_CODE", False) == "1" + # Set GALAXY_DOCS_SKIP_SOURCE=1 to skip building source information and # just build primary documentation. (Quicker to debug issues in most frequently updated # docs). @@ -43,7 +47,9 @@ sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pa extensions = ['recommonmark', 'sphinx.ext.intersphinx', 'sphinx_markdown_tables'] if not SKIP_SOURCE: # TODO: Add https://pypi.org/project/sphinx-autodoc-typehints - extensions += ['sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode', 'sphinx.ext.autodoc'] + extensions += ['sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.autodoc'] + if not SKIP_VIEW_CODE: + extensions.append('sphinx.ext.viewcode') # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] From 08110e2281f664cfdd580c159f91c6356b58207f Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 11 Jan 2021 16:58:41 -0500 Subject: [PATCH 065/208] Better docs & typing for lib/galaxy_test/base/api_asserts.py. --- lib/galaxy_test/base/api_asserts.py | 66 +++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 18 deletions(-) diff --git a/lib/galaxy_test/base/api_asserts.py b/lib/galaxy_test/base/api_asserts.py index 3a0071dd6ef..f5474dedad2 100644 --- a/lib/galaxy_test/base/api_asserts.py +++ b/lib/galaxy_test/base/api_asserts.py @@ -1,50 +1,72 @@ -""" Utility methods for making assertions about Galaxy API responses, etc... +"""Utility methods for making assertions about Galaxy API responses, etc... """ +from typing import Any, cast, Dict, Union + +from requests import Response + ASSERT_FAIL_ERROR_CODE = "Expected Galaxy error code %d, obtained %d" ASSERT_FAIL_STATUS_CODE = "Request status code (%d) was not expected value %s. Body was %s" -def assert_status_code_is(response, expected_status_code): +def assert_status_code_is(response: Response, expected_status_code: int): + """Assert that the supplied response has the expect status code.""" response_status_code = response.status_code if expected_status_code != response_status_code: _report_status_code_error(response, expected_status_code) -def assert_status_code_is_ok(response): +def assert_status_code_is_ok(response: Response): + """Assert that the supplied response is okay. + + The easier alternative ``response.raise_for_status()`` might be + perferable generally. + """ response_status_code = response.status_code is_two_hundred_status_code = response_status_code >= 200 and response_status_code <= 300 if not is_two_hundred_status_code: _report_status_code_error(response, "2XX") -def _report_status_code_error(response, expected_status_code): +def _report_status_code_error(response: Response, expected_status_code: Union[str, int]): try: body = response.json() except Exception: - body = "INVALID JSON RESPONSE <%s>" % response.content + body = f"INVALID JSON RESPONSE <{response.text}>" assertion_message = ASSERT_FAIL_STATUS_CODE % (response.status_code, expected_status_code, body) raise AssertionError(assertion_message) -def assert_has_keys(response, *keys): +def assert_has_keys(response: dict, *keys: str): + """Assert that the supplied response (dict) has the supplied keys.""" for key in keys: assert key in response, f"Response [{response}] does not contain key [{key}]" -def assert_not_has_keys(response, *keys): +def assert_not_has_keys(response: dict, *keys: str): + """Assert that the supplied response (dict) does not have the supplied keys.""" for key in keys: assert key not in response, f"Response [{response}] contains invalid key [{key}]" -def assert_error_code_is(response, error_code): - if hasattr(response, "json"): - response = response.json() - assert_has_keys(response, "err_code") - err_code = response["err_code"] +def assert_error_code_is(response: Union[Response, dict], error_code: int): + """Assert that the supplied response has the supplied Galaxy error code. + + Galaxy error codes can be imported from :py:mod:`galaxy.exceptions.error_codes` + to test against. + + :: + + from galaxy.exceptions import error_codes + assert_error_code_is(response, error_codes.USER_REQUEST_MISSING_PARAMETER) + + """ + as_dict = _as_dict(response) + assert_has_keys(as_dict, "err_code") + err_code = as_dict["err_code"] assert err_code == int(error_code), ASSERT_FAIL_ERROR_CODE % (err_code, int(error_code)) -def assert_object_id_error(response): +def assert_object_id_error(response: Response): # for tests that use fake object IDs - API might throw MalformedId (400) or # or ObjectNotFound (404) - depending if the ID happens to be parseable with # servers API key. @@ -56,12 +78,20 @@ def assert_object_id_error(response): assert_error_code_is(response, 404001) -def assert_error_message_contains(response, expected_contains): - if hasattr(response, "json"): - response = response.json() - assert_has_keys(response, "err_msg") - err_msg = response["err_msg"] +def assert_error_message_contains(response: Union[Response, dict], expected_contains: str): + as_dict = _as_dict(response) + assert_has_keys(as_dict, "err_msg") + err_msg = as_dict["err_msg"] assert expected_contains in err_msg +def _as_dict(response: Union[Response, dict]) -> Dict[str, Any]: + as_dict : Dict[str, Any] + if isinstance(response, Response): + as_dict = cast(dict, response.json()) + else: + as_dict = response + return as_dict + + assert_has_key = assert_has_keys From 61188ca27bad0ec9bc5f96c47a7f16cd17138d6b Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 11 Jan 2021 16:58:23 -0500 Subject: [PATCH 066/208] Hook requests up to intersphinx. --- doc/source/conf.py | 5 ++++- lib/galaxy_test/base/api_asserts.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index b2add06a301..c983beef6a1 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -140,7 +140,10 @@ pygments_style = 'sphinx' #modindex_common_prefix = [] # Intersphinx mapping to Python documentation -intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} +intersphinx_mapping = { + 'python': ('https://docs.python.org/3', None), + 'requests': ("https://requests.readthedocs.io/en/master/", None), +} # -- Options for HTML output --------------------------------------------------- diff --git a/lib/galaxy_test/base/api_asserts.py b/lib/galaxy_test/base/api_asserts.py index f5474dedad2..e6d425d3143 100644 --- a/lib/galaxy_test/base/api_asserts.py +++ b/lib/galaxy_test/base/api_asserts.py @@ -20,6 +20,8 @@ def assert_status_code_is_ok(response: Response): The easier alternative ``response.raise_for_status()`` might be perferable generally. + + .. seealso:: :py:meth:`requests.Response.raise_for_status()` """ response_status_code = response.status_code is_two_hundred_status_code = response_status_code >= 200 and response_status_code <= 300 From 9bd66edc0b521e90406a32c926a1f50014006126 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 11 Jan 2021 19:49:45 -0500 Subject: [PATCH 067/208] Re-do API documentation to avoid duplicate labels. --- .gitignore | 2 -- doc/Makefile | 14 +--------- doc/source/api/api.rst | 5 ++++ doc/source/api/ts_api.rst | 5 ++++ doc/source/lib/galaxy.schema.rst | 26 +++++++++++++++++++ doc/source/lib/galaxy.tool_util.client.rst | 18 +++++++++++++ .../lib/galaxy.web.legacy_framework.rst | 18 +++++++++++++ doc/source/lib/galaxy.workflow.refactor.rst | 26 +++++++++++++++++++ 8 files changed, 99 insertions(+), 15 deletions(-) create mode 100644 doc/source/api/api.rst create mode 100644 doc/source/api/ts_api.rst create mode 100644 doc/source/lib/galaxy.schema.rst create mode 100644 doc/source/lib/galaxy.tool_util.client.rst create mode 100644 doc/source/lib/galaxy.web.legacy_framework.rst create mode 100644 doc/source/lib/galaxy.workflow.refactor.rst diff --git a/.gitignore b/.gitignore index ea4673509d6..92e422f8ae9 100644 --- a/.gitignore +++ b/.gitignore @@ -162,8 +162,6 @@ client/galaxy doc/build doc/schema.md doc/source/admin/config_logging_default_yaml.rst -doc/source/api/api.rst -doc/source/api/ts_api.rst doc/source/dev/schema.md doc/source/dev/schema.rst client/styleguide diff --git a/doc/Makefile b/doc/Makefile index 66e184a4e1b..62ef91ba813 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -15,7 +15,7 @@ ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) sou # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source -GENERATED_RST = source/api/api.rst source/api/ts_api.rst source/dev/schema.md source/admin/config_logging_default_yaml.rst +GENERATED_RST = source/dev/schema.md source/admin/config_logging_default_yaml.rst .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext updaterst @@ -48,18 +48,6 @@ source/dev/run_tests_help.txt: source/dev/schema.md: parse_gx_xsd.py schema_template.md ../lib/galaxy/tool_util/xsd/galaxy.xsd ## Build Github-flavored Markdown from Galaxy Tool XSD (expects lxml in environment) python parse_gx_xsd.py schema_template.md ../lib/galaxy/tool_util/xsd/galaxy.xsd > $@ -source/api/api.rst: source/lib/galaxy.webapps.galaxy.api.rst - printf "Galaxy API\n==========\n" > $@ - tail -n +11 source/lib/galaxy.webapps.galaxy.api.rst >> $@ - sed -i.bak -e 's/^galaxy\\\.webapps\\\.galaxy\\\.api\\\.//' $@ - rm -f $@.bak - -source/api/ts_api.rst: source/lib/tool_shed.webapp.api.rst - printf "Tool Shed API\n=============\n" > $@ - tail -n +11 source/lib/tool_shed.webapp.api.rst >> $@ - sed -i.bak -e 's/^tool_shed\\\.webapp\\\.api\\\.//' $@ - rm -f $@.bak - source/admin/config_logging_default_yaml.rst: ../lib/galaxy/config/__init__.py printf '.. code-block:: yaml\n\n' > $@ printf ' galaxy:\n' >> $@ diff --git a/doc/source/api/api.rst b/doc/source/api/api.rst new file mode 100644 index 00000000000..4bb55ccb9c8 --- /dev/null +++ b/doc/source/api/api.rst @@ -0,0 +1,5 @@ +Galaxy API +========== + +For a detailed description of the of Galaxy API controller please see +the API module :py:mod:`galaxy.webapps.galaxy.api`. diff --git a/doc/source/api/ts_api.rst b/doc/source/api/ts_api.rst new file mode 100644 index 00000000000..55ca735002f --- /dev/null +++ b/doc/source/api/ts_api.rst @@ -0,0 +1,5 @@ +ToolShed API +============= + +For a detailed description of the of Tool Shed API controller please see +the API module :py:mod:`tool_shed.webapp.api`. diff --git a/doc/source/lib/galaxy.schema.rst b/doc/source/lib/galaxy.schema.rst new file mode 100644 index 00000000000..38ec22b6fdc --- /dev/null +++ b/doc/source/lib/galaxy.schema.rst @@ -0,0 +1,26 @@ +galaxy.schema package +===================== + +.. automodule:: galaxy.schema + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +galaxy.schema.fields module +--------------------------- + +.. automodule:: galaxy.schema.fields + :members: + :undoc-members: + :show-inheritance: + +galaxy.schema.schema module +--------------------------- + +.. automodule:: galaxy.schema.schema + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/lib/galaxy.tool_util.client.rst b/doc/source/lib/galaxy.tool_util.client.rst new file mode 100644 index 00000000000..714007ba211 --- /dev/null +++ b/doc/source/lib/galaxy.tool_util.client.rst @@ -0,0 +1,18 @@ +galaxy.tool\_util.client package +================================ + +.. automodule:: galaxy.tool_util.client + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +galaxy.tool\_util.client.staging module +--------------------------------------- + +.. automodule:: galaxy.tool_util.client.staging + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/lib/galaxy.web.legacy_framework.rst b/doc/source/lib/galaxy.web.legacy_framework.rst new file mode 100644 index 00000000000..ca7a526215e --- /dev/null +++ b/doc/source/lib/galaxy.web.legacy_framework.rst @@ -0,0 +1,18 @@ +galaxy.web.legacy\_framework package +==================================== + +.. automodule:: galaxy.web.legacy_framework + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +galaxy.web.legacy\_framework.grids module +----------------------------------------- + +.. automodule:: galaxy.web.legacy_framework.grids + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/lib/galaxy.workflow.refactor.rst b/doc/source/lib/galaxy.workflow.refactor.rst new file mode 100644 index 00000000000..c498d156294 --- /dev/null +++ b/doc/source/lib/galaxy.workflow.refactor.rst @@ -0,0 +1,26 @@ +galaxy.workflow.refactor package +================================ + +.. automodule:: galaxy.workflow.refactor + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +galaxy.workflow.refactor.execute module +--------------------------------------- + +.. automodule:: galaxy.workflow.refactor.execute + :members: + :undoc-members: + :show-inheritance: + +galaxy.workflow.refactor.schema module +-------------------------------------- + +.. automodule:: galaxy.workflow.refactor.schema + :members: + :undoc-members: + :show-inheritance: From f5648e20255b5dccb46c46bfd7381706c8f4c634 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 11 Jan 2021 19:50:10 -0500 Subject: [PATCH 068/208] Fixup some linking between documents in sphinx. --- doc/source/admin/cluster.md | 4 ++-- doc/source/admin/nginx.md | 2 +- doc/source/admin/scaling.md | 4 ++-- doc/source/admin/special_topics/index.rst | 1 + 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/source/admin/cluster.md b/doc/source/admin/cluster.md index e61bf49e451..7f9060a9cbe 100644 --- a/doc/source/admin/cluster.md +++ b/doc/source/admin/cluster.md @@ -2,7 +2,7 @@ Galaxy is designed to run jobs on your local system by default, but it can be configured to run jobs on a cluster. The front-end Galaxy application runs on a single server as usual, but tools are run on cluster nodes instead. -A [general reference for the job configuration file](jobs.md) is also available. +A [general reference for the job configuration file](./jobs.md) is also available. ## Distributed Resources Managers @@ -70,7 +70,7 @@ You may also find that attribute caching in your filesystem causes problems with ## Runner Configuration -**This documentation covers configuration of the various runner plugins, not how to distribute jobs to the various plugins.** Consult the [job configuration file documentation](jobs.md) for full details on the correct syntax, and for instructions on how to configure tools to actually use the runners explained below. +**This documentation covers configuration of the various runner plugins, not how to distribute jobs to the various plugins.** Consult the [job configuration file documentation](./jobs.md) for full details on the correct syntax, and for instructions on how to configure tools to actually use the runners explained below. ### Local diff --git a/doc/source/admin/nginx.md b/doc/source/admin/nginx.md index 95adad98664..917567f6b88 100644 --- a/doc/source/admin/nginx.md +++ b/doc/source/admin/nginx.md @@ -399,7 +399,7 @@ these paths need to exposed in the same way. You may find it useful to require authentication for access to certain paths on your server. For example, Galaxy can run a separate reports app which gives useful information about your Galaxy instance. See the [Reports Configuration -documentation](reports) and [Peter Briggs' blog post on the +documentation](./reports) and [Peter Briggs' blog post on the subject](http://galacticengineer.blogspot.com/2015/06/exposing-galaxy-reports-via-nginx-in.html) for more. After successfully following the blog post, Galaxy reports should be available at e.g. `https://galaxy.example.org/reports`. diff --git a/doc/source/admin/scaling.md b/doc/source/admin/scaling.md index e50aa6963ec..71f35ef2f9c 100644 --- a/doc/source/admin/scaling.md +++ b/doc/source/admin/scaling.md @@ -438,8 +438,8 @@ separated, to the `job-handlers` farm. For example, 3 handlers are defined like By default, a job will be handled by whatever mule currently has the lock on the mule message queue. After receiving a message, it will release the lock, giving other mules a chance to handle future jobs. Jobs can be explicitly mapped to -specific mules as described in the [Job configuration documentation](jobs.md) by using the handler IDs -`main.job-handlers.N`, where `N` is the mule's position in the farm, starting at 1 and incrementing for each mule in the +specific mules as described in the [Galaxy Job Configuration](./jobs.md) documentation +by using the handler IDs `main.job-handlers.N`, where `N` is the mule's position in the farm, starting at 1 and incrementing for each mule in the farm (this is not necessarily the mule ID, but it will be if you only define one farm and you add mules to that farm in sequential order). Each worker that you wish to explicitly map jobs to should be defined in the `` section of `job_conf.xml`. *Do not* define a default handler. diff --git a/doc/source/admin/special_topics/index.rst b/doc/source/admin/special_topics/index.rst index f2a33965e2e..a49416a3785 100644 --- a/doc/source/admin/special_topics/index.rst +++ b/doc/source/admin/special_topics/index.rst @@ -10,6 +10,7 @@ Special Topics interactive_environments mulled_containers grt + gtn job_metrics webhooks performance_tracking From e643c09806616a067bbfb050b82fe8f97fa622b0 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 11 Jan 2021 19:50:33 -0500 Subject: [PATCH 069/208] Fixup more sphinx warning in config option doc generation. --- doc/source/admin/galaxy_options.rst | 18 +++++++++--------- lib/galaxy/config/config_manage.py | 2 ++ lib/galaxy/config/sample/galaxy.yml.sample | 4 ++-- lib/galaxy/webapps/galaxy/config_schema.yml | 4 ++-- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 4d387d9e615..336c57ec9b5 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -677,8 +677,8 @@ :Description: involucro is a tool used to build Docker or Singularity containers for tools from Conda dependencies referenced in tools as - `requirement`s. The following path is the location of involucro on - the Galaxy host. This is ignored if the relevant container + `requirement` s. The following path is the location of involucro + on the Galaxy host. This is ignored if the relevant container resolver isn't enabled, and will install on demand unless involucro_auto_init is set to false. The value of this option will be resolved with respect to @@ -3709,7 +3709,7 @@ you can separate Galaxy into multiple processes. There are more than one way to do this, and they are explained in detail in the documentation: - https://docs.galaxyproject.org/en/master/admin/scaling.html + https://docs.galaxyproject.org/en/master/admin/scaling.html By default, Galaxy manages and executes jobs from within a single process and notifies itself of new jobs via in-memory queues. Jobs are run locally on the system on which Galaxy is started. @@ -4085,7 +4085,7 @@ :Description: Prologue Markdown/HTML to apply to markdown exports to PDF. Allowing branded headers. -:Default: ```` +:Default: ``""`` :Type: str @@ -4096,7 +4096,7 @@ :Description: Prologue Markdown/HTML to apply to markdown exports to PDF. Allowing branded footers. -:Default: ```` +:Default: ``""`` :Type: str @@ -4107,7 +4107,7 @@ :Description: Alternative to markdown_export_prologue that applies just to page exports. -:Default: ```` +:Default: ``""`` :Type: str @@ -4118,7 +4118,7 @@ :Description: Alternative to markdown_export_prologue that applies just to invocation report exports. -:Default: ```` +:Default: ``""`` :Type: str @@ -4129,7 +4129,7 @@ :Description: Alternative to markdown_export_epilogue that applies just to page exports. -:Default: ```` +:Default: ``""`` :Type: str @@ -4140,7 +4140,7 @@ :Description: Alternative to markdown_export_epilogue that applies just to invocation report exports. -:Default: ```` +:Default: ``""`` :Type: str diff --git a/lib/galaxy/config/config_manage.py b/lib/galaxy/config/config_manage.py index 0bbaa64977e..238c56cf9d8 100644 --- a/lib/galaxy/config/config_manage.py +++ b/lib/galaxy/config/config_manage.py @@ -426,6 +426,8 @@ def _write_option_rst(args, rst, key, heading_level, option_value): default = "true" elif default is False: default = "false" + elif default == "": + default = '""' rst.write(":Default: ``%s``\n" % default) if type: rst.write(":Type: %s\n" % type) diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index d0e86061f8f..7aa1ff8723a 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -436,7 +436,7 @@ galaxy: # involucro is a tool used to build Docker or Singularity containers # for tools from Conda dependencies referenced in tools as - # `requirement`s. The following path is the location of involucro on + # `requirement` s. The following path is the location of involucro on # the Galaxy host. This is ignored if the relevant container resolver # isn't enabled, and will install on demand unless involucro_auto_init # is set to false. @@ -1830,7 +1830,7 @@ galaxy: # can separate Galaxy into multiple processes. There are more than # one way to do this, and they are explained in detail in the # documentation: - # https://docs.galaxyproject.org/en/master/admin/scaling.html + # https://docs.galaxyproject.org/en/master/admin/scaling.html # By default, Galaxy manages and executes jobs from within a single # process and notifies itself of new jobs via in-memory queues. Jobs # are run locally on the system on which Galaxy is started. Advanced diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 250a771a41d..7abb5892c01 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -505,7 +505,7 @@ mapping: required: false desc: | involucro is a tool used to build Docker or Singularity containers for tools from Conda - dependencies referenced in tools as `requirement`s. The following path is + dependencies referenced in tools as `requirement` s. The following path is the location of involucro on the Galaxy host. This is ignored if the relevant container resolver isn't enabled, and will install on demand unless involucro_auto_init is set to false. @@ -2704,7 +2704,7 @@ mapping: separate Galaxy into multiple processes. There are more than one way to do this, and they are explained in detail in the documentation: - https://docs.galaxyproject.org/en/master/admin/scaling.html + https://docs.galaxyproject.org/en/master/admin/scaling.html By default, Galaxy manages and executes jobs from within a single process and notifies itself of new jobs via in-memory queues. Jobs are run locally on From 6c79073eb4969ac6c3b5220cdec800cbe521a7dd Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 11 Jan 2021 19:51:12 -0500 Subject: [PATCH 070/208] Mark some Sphinx as orphans - they seem to be not linked to TOC for reasons. --- doc/source/admin/mq.md | 4 ++++ doc/source/admin/proxy_package_layout.rst | 2 ++ doc/source/releases/20.09-config.rst | 2 ++ 3 files changed, 8 insertions(+) diff --git a/doc/source/admin/mq.md b/doc/source/admin/mq.md index b4167185898..cfe171a1564 100644 --- a/doc/source/admin/mq.md +++ b/doc/source/admin/mq.md @@ -1,3 +1,7 @@ +```eval_rst +:orphan: +``` + # Message queue Galaxy uses a message queue internally for communicating between processes. diff --git a/doc/source/admin/proxy_package_layout.rst b/doc/source/admin/proxy_package_layout.rst index 7c038adf31f..1cde42234eb 100644 --- a/doc/source/admin/proxy_package_layout.rst +++ b/doc/source/admin/proxy_package_layout.rst @@ -1,3 +1,5 @@ +:orphan: + Proxy Package Layouts ======================================== diff --git a/doc/source/releases/20.09-config.rst b/doc/source/releases/20.09-config.rst index be7cb7d3852..e2657595b03 100644 --- a/doc/source/releases/20.09-config.rst +++ b/doc/source/releases/20.09-config.rst @@ -1,3 +1,5 @@ +:orphan: + Configuration Changes ===================== From eef17877d18367420987f40e1916fabc5ab248f3 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 11 Jan 2021 19:52:09 -0500 Subject: [PATCH 071/208] Link to user announcements to eliminate sphinx warnings. --- doc/source/releases/index.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/source/releases/index.rst b/doc/source/releases/index.rst index 81c8b7c1978..9126bc636ec 100644 --- a/doc/source/releases/index.rst +++ b/doc/source/releases/index.rst @@ -37,3 +37,15 @@ Releases 13.02_announce 13.01_announce older_releases + +User Announcements + +.. toctree:: + :maxdepth: 1 + + 20.09_announce_user + 20.05_announce_user + 20.01_announce_user + 19.09_announce_user + 19.05_announce_user + 19.01_announce_user From f405c61596f25264ba34d2094c3bf3ba6ca7482f Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 11 Jan 2021 19:53:17 -0500 Subject: [PATCH 072/208] Various sphinx fixes. --- lib/galaxy/containers/__init__.py | 21 +++++++------------ lib/galaxy/managers/libraries.py | 2 +- lib/galaxy/visualization/plugins/registry.py | 2 +- lib/galaxy/web/framework/middleware/batch.py | 1 + lib/galaxy/webapps/base/controller.py | 1 + lib/galaxy/webapps/base/webapp.py | 5 ++++- lib/galaxy/webapps/galaxy/api/authenticate.py | 1 - .../galaxy/api/tool_shed_repositories.py | 5 ++--- lib/galaxy/webapps/galaxy/api/tools.py | 18 ++++++++-------- lib/tool_shed/managers/groups.py | 2 +- lib/tool_shed/managers/repositories.py | 2 +- lib/tool_shed/webapp/api/authenticate.py | 3 +-- 12 files changed, 29 insertions(+), 34 deletions(-) diff --git a/lib/galaxy/containers/__init__.py b/lib/galaxy/containers/__init__.py index 2d12bd9e06e..5c690c6c3d1 100644 --- a/lib/galaxy/containers/__init__.py +++ b/lib/galaxy/containers/__init__.py @@ -14,8 +14,7 @@ from abc import ( abstractmethod, abstractproperty ) -from collections import namedtuple -from typing import Any, Dict, Optional, Type +from typing import Any, Dict, NamedTuple, Optional, Type import yaml @@ -29,18 +28,12 @@ DEFAULT_CONF = {'_default_': {'type': DEFAULT_CONTAINER_TYPE}} log = logging.getLogger(__name__) -class ContainerPort(namedtuple('ContainerPort', ('port', 'protocol', 'hostaddr', 'hostport'))): - """Named tuple representing ports published by a container, with attributes: - - :ivar port: Port number (inside the container) - :vartype port: int - :ivar protocol: Port protocol, either ``tcp`` or ``udp`` - :vartype protocol: str - :ivar hostaddr: Address or hostname where the published port can be accessed - :vartype hostaddr: str - :ivar hostport: Published port number on which the container can be accessed - :vartype hostport: int - """ +class ContainerPort(NamedTuple): + """Named tuple representing ports published by a container, with attributes""" + port: int # Port number (inside the container) + protocol: str # Port protocol, either ``tcp`` or ``udp`` + hostaddr: str # Address or hostname where the published port can be accessed + hostport: int # Published port number on which the container can be accessed class ContainerVolume(metaclass=ABCMeta): diff --git a/lib/galaxy/managers/libraries.py b/lib/galaxy/managers/libraries.py index 141240e10bf..53544a21e85 100644 --- a/lib/galaxy/managers/libraries.py +++ b/lib/galaxy/managers/libraries.py @@ -174,7 +174,7 @@ class LibraryManager: :type check_accessible: bool :returns: the original library - :rtype: Library + :rtype: galaxy.model.Library """ # all libraries are accessible to an admin if trans.user_is_admin: diff --git a/lib/galaxy/visualization/plugins/registry.py b/lib/galaxy/visualization/plugins/registry.py index 4f1dc1cea9f..12460ef24c1 100644 --- a/lib/galaxy/visualization/plugins/registry.py +++ b/lib/galaxy/visualization/plugins/registry.py @@ -51,7 +51,7 @@ class VisualizationsRegistry: """ Set up the manager and load all visualization plugins. - :type app: UniverseApplication + :type app: galaxy.app.UniverseApplication :param app: the application (and its configuration) using this manager :type base_url: string :param base_url: url to prefix all plugin urls with diff --git a/lib/galaxy/web/framework/middleware/batch.py b/lib/galaxy/web/framework/middleware/batch.py index 722e8bb8da4..30e7284b8d0 100644 --- a/lib/galaxy/web/framework/middleware/batch.py +++ b/lib/galaxy/web/framework/middleware/batch.py @@ -2,6 +2,7 @@ Batch API middleware Adds a single route to the installation that: + 1. accepts a POST call containing a JSON array of 'http-like' JSON dictionaries. 2. Each dictionary describes a single API call within the batch and is routed diff --git a/lib/galaxy/webapps/base/controller.py b/lib/galaxy/webapps/base/controller.py index 07d49494552..fb4e19cc463 100644 --- a/lib/galaxy/webapps/base/controller.py +++ b/lib/galaxy/webapps/base/controller.py @@ -350,6 +350,7 @@ class JSAppLauncher(BaseUIController): `masthead` (boolean): (optional, default=True) include masthead elements in the initial page dom. `additional_options` (kwargs): update to the options sent to the app. + """ options = options or self._get_js_options(trans) options.update(additional_options) diff --git a/lib/galaxy/webapps/base/webapp.py b/lib/galaxy/webapps/base/webapp.py index 2fab4d18435..a9851d41ba8 100644 --- a/lib/galaxy/webapps/base/webapp.py +++ b/lib/galaxy/webapps/base/webapp.py @@ -70,8 +70,9 @@ class WebApplication(base.WebApplication): Base WSGI application instantiated for all Galaxy webapps. A web application that: + * adds API and UI controllers by scanning given directories and - importing all modules found there. + importing all modules found there. * has a security object. * builds mako template lookups. * generates GalaxyWebTransactions. @@ -765,8 +766,10 @@ class GalaxyWebTransaction(base.DefaultWebTransaction, def get_history(self, create=False, most_recent=False): """ Load the current history. + - If that isn't available, we find the most recently updated history. - If *that* isn't available, we get or create the default history. + Transactions will not always have an active history (API requests), so None is a valid response. """ diff --git a/lib/galaxy/webapps/galaxy/api/authenticate.py b/lib/galaxy/webapps/galaxy/api/authenticate.py index df34bc3ddc6..e5c902521f0 100644 --- a/lib/galaxy/webapps/galaxy/api/authenticate.py +++ b/lib/galaxy/webapps/galaxy/api/authenticate.py @@ -8,7 +8,6 @@ Sample usage Returns -.. highlight:: json .. code-block:: json { diff --git a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py index d15a9bd4188..044e8996a50 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -420,10 +420,9 @@ class ToolShedRepositoriesController(BaseAPIController): """ GET /api/tool_shed_repositories/{encoded_tool_shed_repsository_id} - Display a dictionary containing information about a specified tool_shed_repository + Display a dictionary containing information about a specified tool_shed_repository. - .. highlight:: json - .. code-block:: json + .. code-block:: { id: (string) Galaxy ID diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index b5cc2992f76..274903f4604 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -38,17 +38,17 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin): @expose_api_anonymous_and_sessionless def index(self, trans, **kwds): """ - GET /api/tools: returns a list of tools defined by parameters:: + GET /api/tools - parameters: + returns a list of tools defined by parameters - in_panel - if true, tools are returned in panel structure, - including sections and labels - trackster - if true, only tools that are compatible with - Trackster are returned - q - if present search on the given query will be performed - tool_id - if present the given tool_id will be searched for - all installed versions + :param in_panel: if true, tools are returned in panel structure, + including sections and labels + :param trackster: if true, only tools that are compatible with + Trackster are returned + :param q: if present search on the given query will be performed + :param tool_id: if present the given tool_id will be searched for + all installed versions """ # Read params. diff --git a/lib/tool_shed/managers/groups.py b/lib/tool_shed/managers/groups.py index b6093479cb3..45b3104e2f1 100644 --- a/lib/tool_shed/managers/groups.py +++ b/lib/tool_shed/managers/groups.py @@ -34,7 +34,7 @@ class GroupManager: :type decoded_group_id: int :returns: the requested group - :rtype: Group + :rtype: tool_shed.model.Group """ if decoded_group_id is None and name is None: raise RequestParameterInvalidException('You must supply either ID or a name of the group.') diff --git a/lib/tool_shed/managers/repositories.py b/lib/tool_shed/managers/repositories.py index 9efcce4e270..d1734251c3e 100644 --- a/lib/tool_shed/managers/repositories.py +++ b/lib/tool_shed/managers/repositories.py @@ -28,7 +28,7 @@ class RepoManager: :type decoded_repo_id: int :returns: the requested repo - :rtype: Repository + :rtype: tool_shed.webapp.model.Repository """ try: repo = trans.sa_session.query(trans.app.model.Repository).filter(trans.app.model.Repository.table.c.id == decoded_repo_id).one() diff --git a/lib/tool_shed/webapp/api/authenticate.py b/lib/tool_shed/webapp/api/authenticate.py index 9161107639f..e1a7bb865d6 100644 --- a/lib/tool_shed/webapp/api/authenticate.py +++ b/lib/tool_shed/webapp/api/authenticate.py @@ -8,11 +8,10 @@ Sample usage: Returns -.. highlight:: json .. code-block:: json { - "api_key": + "api_key": "" } """ From 8d06cffae4397307c508a68d0d635c4fe57aa01a Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 11 Jan 2021 22:06:39 -0500 Subject: [PATCH 073/208] Remove now unused .gitignore file. Or am I missing how that file is generated in someway... --- .gitignore | 1 - .../lib/galaxy.webapps.reports.framework.rst | 18 ------------------ 2 files changed, 19 deletions(-) delete mode 100644 doc/source/lib/galaxy.webapps.reports.framework.rst diff --git a/.gitignore b/.gitignore index 92e422f8ae9..5f286aff736 100644 --- a/.gitignore +++ b/.gitignore @@ -163,7 +163,6 @@ doc/build doc/schema.md doc/source/admin/config_logging_default_yaml.rst doc/source/dev/schema.md -doc/source/dev/schema.rst client/styleguide # webpack stats diff --git a/doc/source/lib/galaxy.webapps.reports.framework.rst b/doc/source/lib/galaxy.webapps.reports.framework.rst deleted file mode 100644 index 1a39715bd9e..00000000000 --- a/doc/source/lib/galaxy.webapps.reports.framework.rst +++ /dev/null @@ -1,18 +0,0 @@ -galaxy.webapps.reports.framework package -======================================== - -.. automodule:: galaxy.webapps.reports.framework - :members: - :undoc-members: - :show-inheritance: - -Submodules ----------- - -galaxy.webapps.reports.framework.grids module ---------------------------------------------- - -.. automodule:: galaxy.webapps.reports.framework.grids - :members: - :undoc-members: - :show-inheritance: From f674613f8fa65830cffa8bc9d9344f7ab09c16c2 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 11 Jan 2021 22:25:24 -0500 Subject: [PATCH 074/208] Allow setting sphinxopts via environment variables (-j 8 is fun) --- doc/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Makefile b/doc/Makefile index 62ef91ba813..de5921267ad 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -2,7 +2,7 @@ # # You can set these variables from the command line. -SPHINXOPTS = +SPHINXOPTS ?= SPHINXBUILD = sphinx-build PAPER = BUILDDIR = build From ee3280ad5286bd00c36b2cbf8baa483debb5dee2 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 11 Jan 2021 22:25:49 -0500 Subject: [PATCH 075/208] Mark pre-annoucnement page as orphan. I guess I always imagined we would link these but then I was told we shouldn't - so I don't even know why we're generating them to be honest then. --- doc/source/releases/21.01_announce.rst | 1 + scripts/bootstrap_history.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/doc/source/releases/21.01_announce.rst b/doc/source/releases/21.01_announce.rst index 631ca300a94..87b2347860b 100644 --- a/doc/source/releases/21.01_announce.rst +++ b/doc/source/releases/21.01_announce.rst @@ -1,3 +1,4 @@ +:orphan: =========================================================== January 2021 Galaxy Release (v 21.01) diff --git a/scripts/bootstrap_history.py b/scripts/bootstrap_history.py index 78a0968214c..86229a219d3 100644 --- a/scripts/bootstrap_history.py +++ b/scripts/bootstrap_history.py @@ -172,6 +172,8 @@ Please see the `full release notes <${release}_announce.html>`_ for more details """) NEXT_TEMPLATE = string.Template(""" +:orphan: + =========================================================== ${month_name} 20${year} Galaxy Release (v ${version}) =========================================================== From e3ffa759f3f022e5b5db73979ed74eceec06983b Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 11 Jan 2021 22:28:04 -0500 Subject: [PATCH 076/208] A bunch more sphinx fixes. --- lib/galaxy/authnz/__init__.py | 35 ++-- lib/galaxy/authnz/psa_authnz.py | 8 +- lib/galaxy/containers/__init__.py | 7 +- lib/galaxy/datatypes/dataproviders/base.py | 17 +- lib/galaxy/datatypes/dataproviders/column.py | 35 ++-- lib/galaxy/datatypes/dataproviders/dataset.py | 38 +--- lib/galaxy/datatypes/mothur.py | 74 +++++--- lib/galaxy/datatypes/tabular.py | 24 +-- lib/galaxy/jobs/dynamic_tool_destination.py | 164 +++++++----------- lib/galaxy/jobs/manager.py | 4 +- lib/galaxy/jobs/runners/drmaa.py | 4 + lib/galaxy/jobs/runners/univa.py | 6 +- lib/galaxy/managers/cloud.py | 4 +- lib/galaxy/managers/rbac_secured.py | 2 +- lib/galaxy/model/metadata.py | 7 +- lib/galaxy/selenium/navigates_galaxy.py | 4 +- lib/galaxy/selenium/sizzle.py | 16 +- .../tools/tool_panel_manager.py | 14 +- lib/galaxy/tool_util/deps/mulled/get_tests.py | 2 +- .../resolvers/unlinked_tool_shed_packages.py | 8 +- lib/galaxy/tool_util/parser/output_objects.py | 22 +-- lib/galaxy/tool_util/parser/xml.py | 2 + lib/galaxy/tools/__init__.py | 15 +- lib/galaxy/tools/actions/__init__.py | 12 +- lib/galaxy/tools/data/__init__.py | 21 ++- lib/galaxy/util/properties.py | 1 + .../visualization/data_providers/basic.py | 7 +- .../visualization/data_providers/genome.py | 4 +- .../visualization/plugins/config_parser.py | 2 +- .../plugins/interactive_environments.py | 2 +- lib/galaxy/web/__init__.py | 4 +- lib/galaxy/web/framework/middleware/batch.py | 20 ++- lib/galaxy/web_stack/handlers.py | 2 +- lib/galaxy/webapps/base/controller.py | 19 +- lib/galaxy/webapps/galaxy/api/histories.py | 7 +- lib/galaxy/webapps/util.py | 2 +- lib/galaxy/workflow/steps.py | 4 +- lib/tool_shed/grids/util.py | 4 +- ...st_0440_deleting_dependency_definitions.py | 4 + .../functional/test_1160_tool_help_images.py | 11 +- .../test_1300_reset_all_metadata.py | 3 +- lib/tool_shed/util/shed_util_common.py | 5 +- .../webapp/controllers/repository_review.py | 1 + 43 files changed, 331 insertions(+), 316 deletions(-) diff --git a/lib/galaxy/authnz/__init__.py b/lib/galaxy/authnz/__init__.py index 00fbc428630..7087f96a8f2 100644 --- a/lib/galaxy/authnz/__init__.py +++ b/lib/galaxy/authnz/__init__.py @@ -26,12 +26,14 @@ class IdentityProvider: :type config: lxml.etree.ElementTree._Element :param config: Is the configuration element of the provider - from the configuration file (e.g., oidc_config.xml). - This element contains the all the provider-specific - configuration elements. + from the configuration file (e.g., oidc_config.xml). + This element contains the all the provider-specific + configuration elements. :type backend_config: lxml.etree.ElementTree._Element - :param backend_config: Is the configuration element of the backend of + :param backend_config: + + Is the configuration element of the backend of the provider from the configuration file (e.g., oidc_backends_config.xml). This element contains all the backend-specific configuration elements. @@ -47,24 +49,23 @@ class IdentityProvider: :type trans: GalaxyWebTransaction :param trans: Galaxy web transaction. - :return: a redirect URI to the provider's authentication - endpoint. + :returns: a redirect URI to the provider's authentication endpoint """ raise NotImplementedError() - def callback(self, state_token, authz_code, trans, login_redirect_url): - """ - Handles authentication call-backs from identity providers. - This process maps `state-token` to a user - :type state_token: string + def callback(self, state_token: str, authz_code: str, trans, login_redirect_url): + """Handles authentication call-backs from identity providers. + + This process maps `state-token` to a user. + :param state_token: is an anti-forgery token which identifies - a Galaxy user to whom the given authorization code belongs to. - :type authz_code: string + a Galaxy user to whom the given authorization code belongs to. :param authz_code: a very short-lived, single-use token to - request a refresh token. + request a refresh token. :type trans: GalaxyWebTransaction :param trans: Galaxy web transaction. - :return tuple: a tuple of redirect_url and user. + :rtype: tuple + :returns: a tuple of redirect_url and user. """ raise NotImplementedError() @@ -79,7 +80,7 @@ class IdentityProvider: :type trans: GalaxyWebTransaction :param trans: Galaxy web transaction. - :type trans: string - :param trans: Optional URL to redirect to after logging out of IDP. + :type post_logout_redirect_url: string + :param post_logout_redirect_url: Optional URL to redirect to after logging out of IDP. """ raise NotImplementedError() diff --git a/lib/galaxy/authnz/psa_authnz.py b/lib/galaxy/authnz/psa_authnz.py index 34dbb592181..3114b0b331f 100644 --- a/lib/galaxy/authnz/psa_authnz.py +++ b/lib/galaxy/authnz/psa_authnz.py @@ -282,9 +282,10 @@ def contains_required_data(response=None, is_new=False, **kwargs): and returns void if otherwise. :type response: dict - :param response: a dictionary containing decoded response from - OIDC backend that contain the following keys - among others: + :param response: a dictionary containing decoded response from + OIDC backend that contain the following keys + among others: + - id_token; see: http://openid.net/specs/openid-connect-core-1_0.html#IDToken - access_token; see: https://tools.ietf.org/html/rfc6749#section-1.4 - refresh_token; see: https://tools.ietf.org/html/rfc6749#section-1.5 @@ -297,6 +298,7 @@ def contains_required_data(response=None, is_new=False, **kwargs): :param is_new: has the user been authenticated? :param kwargs: may contain the following keys among others: + - uid: user ID - user: Galaxy user; if user is already authenticated - backend: the backend that is used for user authentication. diff --git a/lib/galaxy/containers/__init__.py b/lib/galaxy/containers/__init__.py index 5c690c6c3d1..9e39d42aebc 100644 --- a/lib/galaxy/containers/__init__.py +++ b/lib/galaxy/containers/__init__.py @@ -73,12 +73,15 @@ class ContainerVolume(metaclass=ABCMeta): class Container(metaclass=ABCMeta): def __init__(self, interface, id, name=None, **kwargs): - """:param interface: Container interface for the given container type + """ + + :param interface: Container interface for the given container type :type interface: :class:`ContainerInterface` subclass instance :param id: Container identifier :type id: str :param name: Container name :type name: str + """ self._interface = interface self._id = id @@ -111,7 +114,7 @@ class Container(metaclass=ABCMeta): """Attribute for accessing details of ports published by the container. :returns: Port details - :rtype: list of :class:`ContainerPort`s + :rtype: list of :class:`ContainerPort` s """ @abstractproperty diff --git a/lib/galaxy/datatypes/dataproviders/base.py b/lib/galaxy/datatypes/dataproviders/base.py index 22cd392d930..4b087170e41 100644 --- a/lib/galaxy/datatypes/dataproviders/base.py +++ b/lib/galaxy/datatypes/dataproviders/base.py @@ -66,10 +66,11 @@ class HasSettings(type): class DataProvider(metaclass=HasSettings): """ Base class for all data providers. Data providers: - (a) have a source (which must be another file-like object) - (b) implement both the iterator and context manager interfaces - (c) do not allow write methods - (but otherwise implement the other file object interface methods) + + - have a source (which must be another file-like object) + - implement both the iterator and context manager interfaces + - do not allow write methods (but otherwise implement the other file object interface methods) + """ # a definition of expected types for keyword arguments sent to __init__ # useful for controlling how query string dictionaries can be parsed into correct types for __init__ @@ -77,10 +78,12 @@ class DataProvider(metaclass=HasSettings): settings: Dict[str, str] = {} def __init__(self, source, **kwargs): - """ + """Sets up a data provider, validates supplied source. + :param source: the source that this iterator will loop over. - (Should implement the iterable interface and ideally have the - context manager interface as well) + (Should implement the iterable interface and ideally have the + context manager interface as well) + """ self.source = self.validate_source(source) diff --git a/lib/galaxy/datatypes/dataproviders/column.py b/lib/galaxy/datatypes/dataproviders/column.py index 1306338714d..616d65aace6 100644 --- a/lib/galaxy/datatypes/dataproviders/column.py +++ b/lib/galaxy/datatypes/dataproviders/column.py @@ -44,41 +44,40 @@ class ColumnarDataProvider(line.RegexLineDataProvider): column_count=None, column_types=None, parsers=None, parse_columns=True, deliminator='\t', filters=None, **kwargs): """ + :param indeces: a list of indeces of columns to gather from each row - Optional: will default to `None`. - If `None`, this provider will return all rows (even when a - particular row contains more/less than others). - If a row/line does not contain an element at a given index, the - provider will-return/fill-with a `None` value as the element. + Optional: will default to `None`. + If `None`, this provider will return all rows (even when a + particular row contains more/less than others). + If a row/line does not contain an element at a given index, the + provider will-return/fill-with a `None` value as the element. :type indeces: list or None :param column_count: an alternate means of defining indeces, use an int - here to effectively provide the first N columns. - Optional: will default to `None`. + here to effectively provide the first N columns. + Optional: will default to `None`. :type column_count: int :param column_types: a list of string names of types that the - provider will use to look up an appropriate parser for the column. - (e.g. 'int', 'float', 'str', 'bool') - Optional: will default to parsing all columns as strings. + provider will use to look up an appropriate parser for the column. + (e.g. 'int', 'float', 'str', 'bool') + Optional: will default to parsing all columns as strings. :type column_types: list of strings :param parsers: a dictionary keyed with column type strings - and with values that are functions to use when parsing those - types. - Optional: will default to using the function `_get_default_parsers`. + and with values that are functions to use when parsing those + types. + Optional: will default to using the function `_get_default_parsers`. :type parsers: dictionary - :param parse_columns: attempt to parse columns? - Optional: defaults to `True`. + :param parse_columns: attempt to parse columns? Optional: defaults to `True`. :type parse_columns: bool - :param deliminator: character(s) used to split each row/line of the source. - Optional: defaults to the tab character. + :param deliminator: character(s) used to split each row/line of the source. Optional: defaults to the tab character. :type deliminator: str .. note:: that the subclass constructors are passed kwargs - so they're - params (limit, offset, etc.) are also applicable here. + params (limit, offset, etc.) are also applicable here. """ # TODO: other columnar formats: csv, etc. super().__init__(source, **kwargs) diff --git a/lib/galaxy/datatypes/dataproviders/dataset.py b/lib/galaxy/datatypes/dataproviders/dataset.py index 687f5f0062a..66e08f460a7 100644 --- a/lib/galaxy/datatypes/dataproviders/dataset.py +++ b/lib/galaxy/datatypes/dataproviders/dataset.py @@ -531,6 +531,7 @@ class BigWigDataProvider(base.LimitedOffsetDataProvider): def __init__(self, source, chrom, start, end, named_columns=False, column_names=None, **kwargs): """ + :param chrom: which chromosome within the bigbed file to extract data for :type chrom: str :param start: the start of the region from which to extract data @@ -539,15 +540,16 @@ class BigWigDataProvider(base.LimitedOffsetDataProvider): :type end: int :param named_columns: optionally return dictionaries keying each column - with 'chrom', 'start', 'end', 'strand', or 'name'. - Optional: defaults to False + with 'chrom', 'start', 'end', 'strand', or 'name'. + Optional: defaults to False :type named_columns: bool :param column_names: an ordered list of strings that will be used as the keys - for each column in the returned dictionaries. - The number of key, value pairs each returned dictionary has will - be as short as the number of column names provided. + for each column in the returned dictionaries. + The number of key, value pairs each returned dictionary has will + be as short as the number of column names provided. :type column_names: + """ raise NotImplementedError('Work in progress') # TODO: validate is a wig @@ -688,32 +690,6 @@ class SamtoolsDataProvider(line.RegexLineDataProvider): return options_dict, new_kwargs -class BcftoolsDataProvider(line.RegexLineDataProvider): - """ - Data provider that uses an bcftools on a bcf (or vcf?) file as its source. - - This can be piped through other providers (column, map, genome region, etc.). - """ - - def __init__(self, dataset, **kwargs): - # TODO: as samtools - raise NotImplementedError() - # super(BcftoolsDataProvider, self).__init__(dataset, **kwargs) - - -class BGzipTabixDataProvider(base.DataProvider): - """ - Data provider that uses an g(un)zip on a file as its source. - - This can be piped through other providers (column, map, genome region, etc.). - """ - - def __init__(self, dataset, **kwargs): - # TODO: as samtools - need more info on output format - raise NotImplementedError() - # super(BGzipTabixDataProvider, self).__init__(dataset, **kwargs) - - class SQliteDataProvider(base.DataProvider): """ Data provider that uses a sqlite database file as its source. diff --git a/lib/galaxy/datatypes/mothur.py b/lib/galaxy/datatypes/mothur.py index 2d495c05a53..94fa6b2a3b7 100644 --- a/lib/galaxy/datatypes/mothur.py +++ b/lib/galaxy/datatypes/mothur.py @@ -357,6 +357,7 @@ class LowerTriangleDistanceMatrix(DistanceMatrix): Determines whether the file is a lower-triangle distance matrix (phylip) format The first line has the number of sequences in the matrix. The remaining lines have the sequence name followed by a list of distances from all preceeding sequences + 5 # possibly but not always preceded by a tab :/ U68589 U68590 0.3371 @@ -422,6 +423,7 @@ class SquareDistanceMatrix(DistanceMatrix): The first line has the number of sequences in the matrix. The following lines have the sequence name in the first column plus a column for the distance to each sequence in the row order in which they appear in the matrix. + 3 U68589 0.0000 0.3371 0.3610 U68590 0.3371 0.0000 0.3783 @@ -623,11 +625,14 @@ class Frequency(Tabular): def sniff_prefix(self, file_prefix): """ Determines whether the file is a frequency tabular format for chimera analysis - #1.14.0 - 0 0.000 - 1 0.000 - ... - 155 0.975 + + .. code-block:: + + #1.14.0 + 0 0.000 + 1 0.000 + ... + 155 0.975 >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.freq' ) @@ -685,10 +690,13 @@ class Quantile(Tabular): def sniff_prefix(self, file_prefix): """ Determines whether the file is a quantiles tabular format for chimera analysis - 1 0 0 0 0 0 0 - 2 0.309198 0.309198 0.37161 0.37161 0.37161 0.37161 - 3 0.510982 0.563213 0.693529 0.858939 1.07442 1.20608 - ... + + .. code-block:: + + 1 0 0 0 0 0 0 + 2 0.309198 0.309198 0.37161 0.37161 0.37161 0.37161 + 3 0.510982 0.563213 0.693529 0.858939 1.07442 1.20608 + ... >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.quan' ) @@ -805,17 +813,26 @@ class RefTaxonomy(Tabular): http://www.mothur.org/wiki/Taxonomy_outline A table with 2 or 3 columns: + - SequenceName - Taxonomy (semicolon-separated taxonomy in descending order) - integer ? + Example: 2-column (http://www.mothur.org/wiki/Taxonomy_outline) - X56533.1 Eukaryota;Alveolata;Ciliophora;Intramacronucleata;Oligohymenophorea;Hymenostomatida;Tetrahymenina;Glaucomidae;Glaucoma; - X97975.1 Eukaryota;Parabasalidea;Trichomonada;Trichomonadida;unclassified_Trichomonadida; - AF052717.1 Eukaryota;Parabasalidea; + + .. code-block:: + + X56533.1 Eukaryota;Alveolata;Ciliophora;Intramacronucleata;Oligohymenophorea;Hymenostomatida;Tetrahymenina;Glaucomidae;Glaucoma; + X97975.1 Eukaryota;Parabasalidea;Trichomonada;Trichomonadida;unclassified_Trichomonadida; + AF052717.1 Eukaryota;Parabasalidea; + Example: 3-column (http://vamps.mbl.edu/resources/databases.php) - v3_AA008 Bacteria;Firmicutes;Bacilli;Lactobacillales;Streptococcaceae;Streptococcus 5 - v3_AA016 Bacteria 120 - v3_AA019 Archaea;Crenarchaeota;Marine_Group_I 1 + + .. code-block:: + + v3_AA008 Bacteria;Firmicutes;Bacilli;Lactobacillales;Streptococcaceae;Streptococcus 5 + v3_AA016 Bacteria 120 + v3_AA019 Archaea;Crenarchaeota;Marine_Group_I 1 >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.ref.taxonomy' ) @@ -882,16 +899,19 @@ class Axes(Tabular): Determines whether the file is an axes format The first line may have column headings. The following lines have the name in the first column plus float columns for each axis. - ==> 98_sq_phylip_amazon.fn.unique.pca.axes <== - group axis1 axis2 - forest 0.000000 0.145743 - pasture 0.145743 0.000000 - ==> 98_sq_phylip_amazon.nmds.axes <== - axis1 axis2 - U68589 0.262608 -0.077498 - U68590 0.027118 0.195197 - U68591 0.329854 0.014395 + .. code-block:: + + group axis1 axis2 + forest 0.000000 0.145743 + pasture 0.145743 0.000000 + + .. code-block:: + + axis1 axis2 + U68589 0.262608 -0.077498 + U68590 0.027118 0.195197 + U68591 0.329854 0.014395 >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.axes' ) @@ -943,14 +963,20 @@ class SffFlow(Tabular): https://mothur.org/wiki/flow_file/ The first line is the total number of flow values - 800 for Titanium data. For GS FLX it would be 400. Following lines contain: + - SequenceName - the number of useable flows as defined by 454's software - the flow intensity for each base going in the order of TACG. + Example: + + .. code-block:: + 800 GQY1XT001CQL4K 85 1.04 0.00 1.00 0.02 0.03 1.02 0.05 ... GQY1XT001CQIRF 84 1.02 0.06 0.98 0.06 0.09 1.05 0.07 ... GQY1XT001CF5YW 88 1.02 0.02 1.01 0.04 0.06 1.02 0.03 ... + """ def __init__(self, **kwd): diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index efcd99cbfcc..e6bf0817a20 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -1238,18 +1238,20 @@ class MatrixMarket(TabularData): suitable for representing sparse matrices. Only nonzero entries need be encoded, and the coordinates of each are given explicitly. - The tabular file format is defined as follows:: + The tabular file format is defined as follows: - %%MatrixMarket matrix coordinate real general <--- header line - % <--+ - % comments |-- 0 or more comment lines - % <--+ - M N L <--- rows, columns, entries - I1 J1 A(I1, J1) <--+ - I2 J2 A(I2, J2) | - I3 J3 A(I3, J3) |-- L lines - . . . | - IL JL A(IL, JL) <--+ + .. code-block:: + + %%MatrixMarket matrix coordinate real general <--- header line + % <--+ + % comments |-- 0 or more comment lines + % <--+ + M N L <--- rows, columns, entries + I1 J1 A(I1, J1) <--+ + I2 J2 A(I2, J2) | + I3 J3 A(I3, J3) |-- L lines + . . . | + IL JL A(IL, JL) <--+ Indices are 1-based, i.e. A(1,1) is the first element. diff --git a/lib/galaxy/jobs/dynamic_tool_destination.py b/lib/galaxy/jobs/dynamic_tool_destination.py index 36832cd5c6a..bc728b1ea8e 100755 --- a/lib/galaxy/jobs/dynamic_tool_destination.py +++ b/lib/galaxy/jobs/dynamic_tool_destination.py @@ -83,21 +83,17 @@ class RuleValidator: """ @classmethod - def validate_rule(cls, rule_type, app, return_bool=False, *args, **kwargs): + def validate_rule(cls, rule_type: str, app, return_bool: bool = False, *args, **kwargs): """ This function is responsible for passing each rule to its relevant function. - @type rule_type: str - @param rule_type: the current rule's type - - @type return_bool: bool - @param return_bool: True when we are only interested in the result of + :param rule_type: the current rule's type + :param return_bool: True when we are only interested in the result of the validation, and not the validated rule itself. - - @rtype: bool, dict (depending on return_bool) - @return: validated rule or result of validation (depending on - return_bool) + :rtype: bool, dict (depending on return_bool) + :returns: validated rule or result of validation (depending on + return_bool) """ if rule_type == 'file_size': return cls.__validate_file_size_rule(app, return_bool, *args, **kwargs) @@ -380,30 +376,20 @@ class RuleValidator: return valid_rule, rule @classmethod - def __validate_destination(cls, valid_rule, app, return_bool, rule, tool, counter): + def __validate_destination(cls, valid_rule: bool, app, return_bool: bool, rule: dict, tool: str, counter: int): """ This function is responsible for validating destination. - @type valid_rule: bool - @param valid_rule: returns True if everything is valid. False if it + :param valid_rule: returns True if everything is valid. False if it encounters any abnormalities in the config. - - @type return_bool: bool - @param return_bool: True when we are only interested in the result of + :param return_bool: True when we are only interested in the result of the validation, and not the validated rule itself. - - @type rule: dict - @param rule: contains the original received rule - - @type tool: str - @param tool: the name of the current tool. Necessary for log output. - - @type counter: int - @param counter: this counter is used to identify what rule # is + :param rule: contains the original received rule + :param tool: the name of the current tool. Necessary for log output. + :param counter: this counter is used to identify what rule # is currently being validated. Necessary for log output. - - @rtype: bool, dict (tuple) - @return: validated rule and result of validation + :rtype: bool, dict (tuple) + :returns: validated rule and result of validation """ if "fail_message" in rule: @@ -698,27 +684,20 @@ class RuleValidator: return valid_rule, rule -def parse_yaml(path="/config/tool_destinations.yml", - job_conf_path="/config/job_conf.xml", app=None, test=False, - return_bool=False): +def parse_yaml(path: str = "/config/tool_destinations.yml", + job_conf_path: str = "/config/job_conf.xml", app=None, test: bool = False, + return_bool: bool = False): """ Get a yaml file from path and send it to validate_config for validation. - @type path: str - @param path: the path to the tool destinations config file - - @type job_conf_path: str - @param job_conf_path: the path to the job config file - - @type test: bool - @param test: indicates whether to run in test mode or production mode - - @type return_bool: bool - @param return_bool: True when we are only interested in the result of the + :param path: the path to the tool destinations config file + :param job_conf_path: the path to the job config file + :param test: indicates whether to run in test mode or production mode + :type return_bool: bool + :param return_bool: True when we are only interested in the result of the validation, and not the validated rule itself. - - @rtype: bool, dict (depending on return_bool) - @return: validated rule or result of validation (depending on return_bool) + :rtype: bool, dict (depending on return_bool) + :returns: validated rule or result of validation (depending on return_bool) """ @@ -754,7 +733,7 @@ def parse_yaml(path="/config/tool_destinations.yml", config = validate_config(config, app) except MalformedYMLException: if verbose: - log.error(str(sys.exc_value)) + log.exception("Failed to parse YAML for dynamic job destination") raise except ScannerError: if verbose: @@ -768,34 +747,26 @@ def parse_yaml(path="/config/tool_destinations.yml", return config -def validate_destination(app, destination, err_message, err_message_contents, - return_bool=True): +def validate_destination(app, destination: str, err_message: str, err_message_contents, + return_bool: bool = True): """ Validate received destination id. - @type app: - @param app: Current app - - @type destination: str - @param destination: string containing the destination id that is being + :param app: Current app + :param destination: string containing the destination id that is being validated - - @type err_message: str - @param err_message: Error message to be formatted with the contents of + :param err_message: Error message to be formatted with the contents of `err_message_contents` upon the event of invalid destination - - @type err_message_contents: tuple - @param err_message_contents: A tuple of strings to be placed in - `err_message` - - @type return_bool: bool - @param return_bool: Whether or not the calling function has been told to + :type err_message_contents: tuple + :param err_message_contents: A tuple of strings to be placed in + ``err_message`` + :param return_bool: Whether or not the calling function has been told to return a boolean value or not. Determines whether or not to print 'Ignoring...' after error messages. - @rtype: bool - @return: True if the destination is valid and False otherwise. + :rtype: bool + :returns: True if the destination is valid and False otherwise. """ valid_destination = False @@ -824,19 +795,15 @@ def validate_destination(app, destination, err_message, err_message_contents, return valid_destination -def validate_config(obj, app=None, return_bool=False,): +def validate_config(obj: dict, app=None, return_bool: bool = False): """ Validate received config. - @type obj: dict - @param obj: the entire contents of the config - - @type return_bool: bool - @param return_bool: True when we are only interested in the result of the - validation, and not the validated rule itself. - - @rtype: bool, dict (depending on return_bool) - @return: validated rule or result of validation (depending on return_bool) + :param obj: the entire contents of the config + :param return_bool: True when we are only interested in the result of the + validation, and not the validated rule itself. + :rtype: bool, dict (depending on return_bool) + :returns: validated rule or result of validation (depending on return_bool) """ global priority_list @@ -1625,21 +1592,22 @@ def map_tool_to_destination( return destination -def get_destination_list_from_job_config(job_config_location): +def get_destination_list_from_job_config(job_config_location) -> set: """ returns A list of all destination IDs declared in the job configuration - @type job_config_location: str - @param job_config_location: The location of the job config file relative - to the galaxy root directory. If NoneType, defaults to - galaxy/config/job_conf.xml, - galaxy/config/job_conf.xml.sample_advanced, or - galaxy/config/job_conf.xml.sample_basic - (first one that exists) + :type job_config_location: str + :param job_config_location: - @rtype: list - @return: A list of all of the destination IDs declared in the job - configuration file. + The location of the job config file relative + to the galaxy root directory. If NoneType, defaults to + galaxy/config/job_conf.xml, + galaxy/config/job_conf.xml.sample_advanced, or + galaxy/config/job_conf.xml.sample_basic + (first one that exists) + + :return: A list of all of the destination IDs declared in the job + configuration file. """ global destination_list @@ -1751,20 +1719,18 @@ def get_typo_correction(typo_str, word_set, max_dist): the same when case is not considered. If there are no appropriate matches, nothing is returned instead. - @type typo_str: str - @param typo_str: The string to be compared + :type typo_str: str + :param typo_str: The string to be compared + :type word_set: set of str + :param word_set: The set of strings to compare to + :type max_dist: int + :param max_dist: the largest allowed edit distance between + the word and the result. If nothing is + within this range, nothing is returned - @type word_set: set of str - @param word_set: The set of strings to compare to - - @type max_dist: int - @param max_dist: the largest allowed edit distance between - the word and the result. If nothing is - within this range, nothing is returned - - @rtype: str or NoneType - @return: The closest matching string, or None, if no strings - being compared to are within max_dist edit distance. + :rtype: str or NoneType + :returns: The closest matching string, or None, if no strings + being compared to are within max_dist edit distance. """ # Start curr_best out as the largest diff --git a/lib/galaxy/jobs/manager.py b/lib/galaxy/jobs/manager.py index 1bcd87c2fcb..0f90e5a9e24 100644 --- a/lib/galaxy/jobs/manager.py +++ b/lib/galaxy/jobs/manager.py @@ -54,7 +54,7 @@ class JobManager: Due to the nature of some handler assignment methods which are wholly DB-based, the enqueue method will flush the job. Callers who create the job typically should not flush the job before handing it off to ``enqueue()``. - If a job handler cannot be assigned, :exception:`ToolExecutionError` is raised. + If a job handler cannot be assigned, py:class:`ToolExecutionError` is raised. :param job: Job to enqueue. :type job: Instance of :class:`galaxy.model.Job`. @@ -62,7 +62,7 @@ class JobManager: :type tool: Instance of :class:`galaxy.tools.Tool`. :raises ToolExecutionError: if a handler was unable to be assigned. - returns: str or None -- Handler ID, tag, or pool assigned to the job. + :returns: str or None -- Handler ID, tag, or pool assigned to the job. """ tool_id = None configured_handler = None diff --git a/lib/galaxy/jobs/runners/drmaa.py b/lib/galaxy/jobs/runners/drmaa.py index b0b7509fda3..d35a1ac71d1 100644 --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -261,13 +261,17 @@ class DRMAAJobRunner(AsynchronousJobRunner): that could happen in this process. to be called from check_watched_items() returns the state or None if exceptions occurred in the latter case the job is appended to new_watched if a + 1 drmaa.InternalException, 2 drmaa.InvalidJobExceptionnot, or 3 drmaa.DrmCommunicationException occurred + (which causes the job to be tested again in the next iteration of check_watched_items) + - the job is finished as errored if any other exception occurs - the job is finished OK or errored after the maximum number of retries depending on the exception + Note that None is returned in all cases where the loop in check_watched_items is to be continued """ diff --git a/lib/galaxy/jobs/runners/univa.py b/lib/galaxy/jobs/runners/univa.py index 0db05fd3de9..742c7669e38 100644 --- a/lib/galaxy/jobs/runners/univa.py +++ b/lib/galaxy/jobs/runners/univa.py @@ -4,9 +4,12 @@ Job control via the DRMAA API / qstat and qacct. Known to work on the UNIVA grid engine. known bugs/problems: + - if a job runs longer than the time limits of the queue two things happen + 1. at the soft limit (s_rt) SIGUSR1 is sent to the job 2. at the hard limit (h_rt) SIGKILL is sent to the job + The second case is covered by the runner -- it's the same mechanism that kills a job when the job time limit is reached. The first case is currently not covered. The good thing is that most programs ignore SIGUSR1. @@ -14,11 +17,12 @@ known bugs/problems: at random. Possible solutions: + - configure the job destinations such that the queue limits are never reached. the time limits can be determined with `for i in `qconf -sql`; do qconf -sq $i | grep -E '(h|s)_rt'; done` - - extend the job runner to cover s_rt cases (some ideas): + * I'm unsure if the programs reaction to SIGUSR1 can be determined easily because it depends on the program. It seems that exit code is in most cases 10. diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index cc0733940a9..e4cb42243e9 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -61,11 +61,11 @@ class CloudManager(sharable.SharableModelManager): :type provider: string :param provider: the name of cloud-based resource provided. A list of supported providers is given in - `SUPPORTED_PROVIDERS` variable. + `SUPPORTED_PROVIDERS` variable. :type credentials: dict :param credentials: a dictionary containing all the credentials required to authenticated to the - specified provider. + specified provider. :rtype: provider specific, e.g., `cloudbridge.cloud.providers.aws.provider.AWSCloudProvider` for AWS. :return: a cloudbridge connection to the specified provider. diff --git a/lib/galaxy/managers/rbac_secured.py b/lib/galaxy/managers/rbac_secured.py index 4ccda187aa3..dfb5db1d6ae 100644 --- a/lib/galaxy/managers/rbac_secured.py +++ b/lib/galaxy/managers/rbac_secured.py @@ -16,7 +16,7 @@ class RBACPermissionFailedException(galaxy.exceptions.InsufficientPermissionsExc class RBACPermission: """ - Base class for wrangling/controlling the permissions ORM models (*Permissions, Roles) + Base class for wrangling/controlling the permissions ORM models (Permissions, Roles) that control which users can perform certain actions on their associated models (Libraries, Datasets). """ diff --git a/lib/galaxy/model/metadata.py b/lib/galaxy/model/metadata.py index c732282b8bc..b570f05af54 100644 --- a/lib/galaxy/model/metadata.py +++ b/lib/galaxy/model/metadata.py @@ -132,12 +132,13 @@ class MetadataCollection(Mapping): def element_is_set(self, name): """ check if the meta data with the given name is set, i.e. + - if the such a metadata actually exists and - if its value differs from no_value - param name the name of the metadata element - return True if the value differes from the no_value - False if its equal of if no metadata with the name is specified + :param name: the name of the metadata element + :returns: True if the value differes from the no_value + False if its equal of if no metadata with the name is specified """ try: meta_val = self.parent._metadata[name] diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py index 7071d0e9fcc..d92f2982a41 100644 --- a/lib/galaxy/selenium/navigates_galaxy.py +++ b/lib/galaxy/selenium/navigates_galaxy.py @@ -112,14 +112,14 @@ class NavigatesGalaxy(HasDriver): In most cases, methods for interacting with Galaxy components that appear in multiple tests or applications should be refactored into this class for now. - Keep in mind that this class is used outside the context of ``TestCase``s as + Keep in mind that this class is used outside the context of ``TestCase`` s as well - so some methods more explicitly related to test data or assertion checking may make more sense in SeleniumTestCase for instance. Some day this class will likely be split up into smaller mixins for particular components of Galaxy, but until that day the best practice is to prefix methods for driving or querying the interface with the name of the component or page - the method operates on. These serve as psedu-namespaces until we decompose this + the method operates on. These serve as pseudo-namespaces until we decompose this class. For instance, the method for clicking an option in the workflow editor is workflow_editor_click_option instead of click_workflow_editor_option. """ diff --git a/lib/galaxy/selenium/sizzle.py b/lib/galaxy/selenium/sizzle.py index 172c6ac7ca2..1e71e9e25f1 100644 --- a/lib/galaxy/selenium/sizzle.py +++ b/lib/galaxy/selenium/sizzle.py @@ -62,13 +62,11 @@ def sizzle_presence_of_selector(selector): return ec -def find_element_by_sizzle(driver, sizzle_selector): +def find_element_by_sizzle(driver, sizzle_selector: str): """ Finds an element by sizzle selector. - :Args: - - sizzle_selector: The sizzle selector to use when finding element. - :Usage: - driver.find_element_by_sizzle('#foo') + + :param sizzle_selector: The sizzle selector to use when finding element. """ elements = driver.find_elements_by_sizzle(sizzle_selector) if elements: @@ -79,13 +77,11 @@ def find_element_by_sizzle(driver, sizzle_selector): ) -def find_elements_by_sizzle(driver, sizzle_selector): +def find_elements_by_sizzle(driver, sizzle_selector: str): """ Finds elements by sizzle selector. - :Args: - - sizzle_selector: The sizzle selector to use when finding elements. - :Usage: - driver.find_elements_by_sizzle('.foo') + + :param sizzle_selector: The sizzle selector to use when finding elements. """ if not _is_sizzle_loaded(driver): _inject_sizzle(driver, SIZZLE_URL, SIZZLE_LOAD_TIMEOUT) diff --git a/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py b/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py index 136eabb6d20..8ddedec95cb 100644 --- a/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -168,11 +168,15 @@ class ToolPanelManager: The intent is to call this method for every tool config in a repository and append each of these as entries to a tool panel dictionary for the repository. This enables each tool to be loaded into a different section in the tool panel. - { : - [{ tool_config : , - id: , - version : , - name : }]} + + .. code-block:: + + { : + [{ tool_config : , + id: , + version : , + name : }]} + """ tool_panel_dict = {} file_name = strip_path(tool_config) diff --git a/lib/galaxy/tool_util/deps/mulled/get_tests.py b/lib/galaxy/tool_util/deps/mulled/get_tests.py index c30630ee80e..a0cf52e5e51 100644 --- a/lib/galaxy/tool_util/deps/mulled/get_tests.py +++ b/lib/galaxy/tool_util/deps/mulled/get_tests.py @@ -162,7 +162,7 @@ def open_recipe_file(file, recipes_path=None, github_repo='bioconda/bioconda-rec def get_alternative_versions(filepath, filename, recipes_path=None, github_repo='bioconda/bioconda-recipes'): """ - Return files that match 'filepath/*/filename' in the bioconda-recipes repository + Return files that match ``filepath/*/filename`` in the bioconda-recipes repository """ if recipes_path: return [n.replace('%s/' % recipes_path, '') for n in glob(f'{recipes_path}/{filepath}/*/{filename}')] diff --git a/lib/galaxy/tool_util/deps/resolvers/unlinked_tool_shed_packages.py b/lib/galaxy/tool_util/deps/resolvers/unlinked_tool_shed_packages.py index af2d8642042..eafa3b685ad 100644 --- a/lib/galaxy/tool_util/deps/resolvers/unlinked_tool_shed_packages.py +++ b/lib/galaxy/tool_util/deps/resolvers/unlinked_tool_shed_packages.py @@ -3,13 +3,15 @@ Backup resolvers for when dependencies can not be loaded from the database. Mainly suited for testing stage. Ideally all dependencies will be stored in the database - when a tool is added from a Tool Shed. +when a tool is added from a Tool Shed. + That should remain the preferred way of locating dependencies. In cases where that is not possible - for example during testing this resolver can act as a backup. +for example during testing this resolver can act as a backup. + This resolver looks not just for manually added dependencies - but also ones added from a Tool Shed. +but also ones added from a Tool Shed. This tool is still under development so the default behaviour could change. It has been tested when placed in the same directory as galaxy_packages.py diff --git a/lib/galaxy/tool_util/parser/output_objects.py b/lib/galaxy/tool_util/parser/output_objects.py index c6084b62e88..1827f2cc710 100644 --- a/lib/galaxy/tool_util/parser/output_objects.py +++ b/lib/galaxy/tool_util/parser/output_objects.py @@ -25,7 +25,7 @@ class ToolOutput(ToolOutputBase): Represents an output datasets produced by a tool. For backward compatibility this behaves as if it were the tuple:: - (format, metadata_source, parent) + (format, metadata_source, parent) """ dict_collection_visible_keys = ['name', 'format', 'label', 'hidden', 'output_type', 'format_source', @@ -123,15 +123,17 @@ class ToolOutputCollection(ToolOutputBase): Represents a HistoryDatasetCollectionAssociation of output datasets produced by a tool. - - - - - - - - - + .. code-block:: + + + + + + + + + + """ dict_collection_visible_keys = ['name', 'format', 'label', 'hidden', 'output_type', 'default_format', 'default_format_source', 'default_metadata_source', 'inherit_format', 'inherit_metadata'] diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py index 957cb05d6ee..e5573820c45 100644 --- a/lib/galaxy/tool_util/parser/xml.py +++ b/lib/galaxy/tool_util/parser/xml.py @@ -443,6 +443,7 @@ class XmlToolSource(ToolSource): parse error handling from command and stdio tag returns list of exit codes, list of regexes + - exit_codes contain all non-zero exit codes (:-1 and 1:) if detect_errors is default (if not legacy), exit_code, or aggressive - the oom_exit_code if given and detect_errors is exit_code @@ -451,6 +452,7 @@ class XmlToolSource(ToolSource): and exit codes derived from the properties of the command tag. thus more specific regexes of the same or more severe error level are triggered first. + """ command_el = self._command_el diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index d2e12068563..928315fe700 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1117,7 +1117,7 @@ class Tool(Dictifiable): def parse_inputs(self, tool_source): """ - Parse the "" element and create appropriate `ToolParameter`s. + Parse the "" element and create appropriate `ToolParameter` s. This implementation supports multiple pages and grouping constructs. """ # Load parameters (optional) @@ -2728,17 +2728,18 @@ class DataManagerTool(OutputParameterJSONTool): history = None return history - def allow_user_access(self, user, attempting_access=True): - """ + def allow_user_access(self, user, attempting_access=True) -> bool: + """Check user access to this tool. + :param user: model object representing user. :type user: galaxy.model.User :param attempting_access: is the user attempting to do something with the - the tool (set false for incidental checks like toolbox - listing) + the tool (set false for incidental checks like toolbox + listing) :type attempting_access: bool - :returns: bool -- Whether the user is allowed to access the tool. - Data Manager tools are only accessible to admins. + :returns: Whether the user is allowed to access the tool. + Data Manager tools are only accessible to admins. """ if super().allow_user_access(user) and self.app.config.is_admin_user(user): return True diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 015a7b93ed5..afb8cf43681 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -785,11 +785,13 @@ class DefaultToolAction: """ This allows to map names of input files to metadata default values. Example: - - - - - + .. code-block:: + + + + + + """ if output.actions: for action in output.actions.actions: diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py index 20dc67e7353..07fe2e82de5 100644 --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -315,7 +315,9 @@ class ToolDataTable: class TabularToolDataTable(ToolDataTable, Dictifiable): """ Data stored in a tabular / separated value format on disk, allows multiple - files to be merged but all must have the same column definitions:: + files to be merged but all must have the same column definitions: + + .. code-block:: xml @@ -798,13 +800,16 @@ class TabularToolDataField(Dictifiable): class RefgenieToolDataTable(TabularToolDataTable): """ Data stored in refgenie -
- - ${__REFGENIE_UUID__} - ${__REFGENIE_GENOME__} - ${__REFGENIE_DISPLAY_NAME__} - ${__REFGENIE_ASSET__} -
+ + .. code-block:: xml + + + + ${__REFGENIE_UUID__} + ${__REFGENIE_GENOME__} + ${__REFGENIE_DISPLAY_NAME__} + ${__REFGENIE_ASSET__} +
""" dict_collection_visible_keys = ['name'] diff --git a/lib/galaxy/util/properties.py b/lib/galaxy/util/properties.py index 76c4557ba47..535426872b5 100644 --- a/lib/galaxy/util/properties.py +++ b/lib/galaxy/util/properties.py @@ -142,6 +142,7 @@ class NicerConfigParser(ConfigParser): self._interpolation = self.InterpolateWrapper(self._interpolation) read_file = getattr(ConfigParser, 'read_file', ConfigParser.readfp) + read_file.__doc__ = "" def defaults(self): """Return the defaults, with their values interpolated (with the diff --git a/lib/galaxy/visualization/data_providers/basic.py b/lib/galaxy/visualization/data_providers/basic.py index c22eac0baf0..3efb47981ff 100644 --- a/lib/galaxy/visualization/data_providers/basic.py +++ b/lib/galaxy/visualization/data_providers/basic.py @@ -6,8 +6,11 @@ from galaxy.datatypes.tabular import Tabular class BaseDataProvider: """ - Base class for data providers. Data providers (a) read and package data from datasets; - and (b) write subsets of data to new datasets. + Base class for data providers. Data providers both: + + - read and package data from datasets + - write subsets of data to new datasets + """ def __init__(self, converted_dataset=None, original_dataset=None, dependencies=None, diff --git a/lib/galaxy/visualization/data_providers/genome.py b/lib/galaxy/visualization/data_providers/genome.py index 57a8e9090d8..8244d4fa258 100644 --- a/lib/galaxy/visualization/data_providers/genome.py +++ b/lib/galaxy/visualization/data_providers/genome.py @@ -586,6 +586,7 @@ class VcfDataProvider(GenomeDataProvider): Payload format: An array of entries for each locus in the file. Each array has the following entries: + 1. GUID (unused) 2. location (0-based) 3. reference base(s) @@ -594,7 +595,8 @@ class VcfDataProvider(GenomeDataProvider): 6. whether variant passed filter 7. sample genotypes -- a single string with samples separated by commas; empty string denotes the reference genotype - 8-end: allele counts for each alternative + 8. allele counts for each alternative + """ col_name_data_attr_mapping: Dict[Union[str, int], Dict] = {'Qual': {'index': 6, 'name': 'Qual'}} diff --git a/lib/galaxy/visualization/plugins/config_parser.py b/lib/galaxy/visualization/plugins/config_parser.py index 3ee7da3a473..4b3b620dedb 100644 --- a/lib/galaxy/visualization/plugins/config_parser.py +++ b/lib/galaxy/visualization/plugins/config_parser.py @@ -164,7 +164,7 @@ class VisualizationsConfigParser: def parse_entry_point(self, xml_tree): """ Parse the config file for an appropriate entry point: a mako template, a script tag, - or an html file, returning as dictionary with: `type`, `file`, and `attr`ibutes of + or an html file, returning as dictionary with: ``type``, ``file``, and ``attr`` (-ibutes) of the element. """ # (older) mako-only syntax: the template to use in rendering the visualization diff --git a/lib/galaxy/visualization/plugins/interactive_environments.py b/lib/galaxy/visualization/plugins/interactive_environments.py index 54afd0db041..ede9942b818 100644 --- a/lib/galaxy/visualization/plugins/interactive_environments.py +++ b/lib/galaxy/visualization/plugins/interactive_environments.py @@ -510,7 +510,7 @@ class InteractiveEnvironmentRequest: :type env_override: dict :param env_override: dictionary of environment variables to add. - :type volumes: list of :class:`galaxy.containers.docker_model.DockerVolume`s + :type volumes: list of py:class:`galaxy.containers.docker_model.DockerVolume` s :param volumes: dictionary of docker volume mounts """ diff --git a/lib/galaxy/web/__init__.py b/lib/galaxy/web/__init__.py index 6ab1bd9218a..696fd07f2b3 100644 --- a/lib/galaxy/web/__init__.py +++ b/lib/galaxy/web/__init__.py @@ -25,10 +25,10 @@ from .framework.decorators import ( require_login, ) -__all__ = ('FormBuilder', 'do_not_cache', 'error', 'expose', 'expose_api', +__all__ = ('do_not_cache', 'error', 'expose', 'expose_api', 'expose_api_anonymous', 'expose_api_anonymous_and_sessionless', 'expose_api_raw', 'expose_api_raw_anonymous', - 'expose_api_raw_anonymous_and_sessionless', 'form', + 'expose_api_raw_anonymous_and_sessionless', 'format_return_as_json', 'httpexceptions', 'json', 'json_pretty', 'legacy_expose_api', 'legacy_expose_api_anonymous', 'legacy_expose_api_raw', 'legacy_expose_api_raw_anonymous', diff --git a/lib/galaxy/web/framework/middleware/batch.py b/lib/galaxy/web/framework/middleware/batch.py index 30e7284b8d0..d48014ce0ce 100644 --- a/lib/galaxy/web/framework/middleware/batch.py +++ b/lib/galaxy/web/framework/middleware/batch.py @@ -41,16 +41,20 @@ class BatchMiddleware: """ Adds a URL endpoint for processing batch API calls formatted as a JSON array of JSON dictionaries. These dictionaries are in the form: - [ - { - "url": "/api/histories", - "type": "POST", - "body": "{ \"name\": \"New History Name\" }" - }, - ... - ] + + .. code-block: json + + [ + { + "url": "/api/histories", + "type": "POST", + "body": "{ \"name\": \"New History Name\" }" + }, + ... + ] where: + * `url` is the url for the API call to be made including any query string * `type` is the HTTP method used (e.g. 'POST', 'PUT') - defaults to 'GET' * `body` is the text body of the request (optional) diff --git a/lib/galaxy/web_stack/handlers.py b/lib/galaxy/web_stack/handlers.py index 042cfc0d518..612d6b54f86 100644 --- a/lib/galaxy/web_stack/handlers.py +++ b/lib/galaxy/web_stack/handlers.py @@ -434,7 +434,7 @@ class ConfiguresHandlers: def assign_handler(self, obj, configured=None, **kwargs): """Set a job handler, flush obj - Called assignment methods should raise :exception:`HandlerAssignmentSkip` to indicate that the next method + Called assignment methods should raise py:class:`HandlerAssignmentSkip` to indicate that the next method should be tried. :param obj: Object to assign a handler to (must be a model object with ``handler`` attribute and diff --git a/lib/galaxy/webapps/base/controller.py b/lib/galaxy/webapps/base/controller.py index fb4e19cc463..aab9eddcf33 100644 --- a/lib/galaxy/webapps/base/controller.py +++ b/lib/galaxy/webapps/base/controller.py @@ -2,6 +2,7 @@ Contains functionality needed in every web interface """ import logging +from typing import Optional from sqlalchemy import true from webob.exc import ( @@ -338,19 +339,17 @@ class JSAppLauncher(BaseUIController): log.exception(exc) return {} - def template(self, trans, app_name, entry_fn='app', options=None, bootstrapped_data=None, masthead=True, **additional_options): + def template(self, trans, app_name: str, entry_fn: str = 'app', options=None, bootstrapped_data: Optional[dict] = None, masthead: Optional[bool] = True, **additional_options): """ Render and return the single page mako template that starts the app. - `app_name` (string): the first portion of the webpack bundle to as the app. - `entry_fn` (string): the name of the window-scope function that starts the - app. Defaults to 'app'. - `bootstrapped_data` (dict): (optional) update containing any more data - the app may need. - `masthead` (boolean): (optional, default=True) include masthead elements in - the initial page dom. - `additional_options` (kwargs): update to the options sent to the app. - + :param app_name: the first portion of the webpack bundle to as the app. + :param entry_fn: the name of the window-scope function that starts the + app. Defaults to 'app'. + :param bootstrapped_data: update containing any more data + the app may need. + :param masthead: include masthead elements in the initial page dom. + :param additional_options: update to the options sent to the app. """ options = options or self._get_js_options(trans) options.update(additional_options) diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index e7fd32d92a8..716a95bb199 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -475,9 +475,10 @@ class HistoriesController(BaseAPIController, ExportsHistoryMixin, ImportsHistory @expose_api def index_exports(self, trans, id): """ - index_exports(self, trans, id) - * GET /api/histories/{id}/exports: - Get history exports. + GET /api/histories/{id}/exports + + Get previous history exports (to links). Effectively returns serialized + JEHA objects. """ return self.history_export_view.get_exports(trans, id) diff --git a/lib/galaxy/webapps/util.py b/lib/galaxy/webapps/util.py index 7d82610aed5..94ca01e864e 100644 --- a/lib/galaxy/webapps/util.py +++ b/lib/galaxy/webapps/util.py @@ -36,7 +36,7 @@ def wrap_if_allowed_or_fail(app, stack, wrap, name=None, args=None, kwargs=None) Arguments are the same as for :func:`wrap_if_allowed`. - Raises :exception:`MiddlewareWrapUnsupported` if the stack does not allow the middleware. + Raises py:class:`MiddlewareWrapUnsupported` if the stack does not allow the middleware. """ name = name or wrap.__name__ if not stack.allowed_middleware(wrap): diff --git a/lib/galaxy/workflow/steps.py b/lib/galaxy/workflow/steps.py index f6d50d712ae..1ff7e7e2bd8 100644 --- a/lib/galaxy/workflow/steps.py +++ b/lib/galaxy/workflow/steps.py @@ -44,8 +44,8 @@ def order_workflow_steps(steps): def edgelist_for_workflow_steps(steps): """ - Create a list of tuples representing edges between ``WorkflowSteps`` based - on associated ``WorkflowStepConnection``s + Create a list of tuples representing edges between ``WorkflowStep`` s based + on associated ``WorkflowStepConnection`` s """ edges = [] steps_to_index = {step: i for i, step in enumerate(steps)} diff --git a/lib/tool_shed/grids/util.py b/lib/tool_shed/grids/util.py index 39b54089472..5b5d4b6d79f 100644 --- a/lib/tool_shed/grids/util.py +++ b/lib/tool_shed/grids/util.py @@ -109,7 +109,7 @@ def get_latest_downloadable_repository_metadata(trans, repository): """ Return the latest downloadable repository_metadata record for the received repository. This will return repositories of type unrestricted as well as types repository_suite_definition and - tool_dependency_definition. + tool_dependency_definition. """ encoded_repository_id = trans.security.encode_id(repository.id) repo = repository.hg_repo @@ -148,7 +148,7 @@ def get_latest_repository_metadata(trans, repository): """ Return the latest repository_metadata record for the received repository if it exists. This will return repositories of type unrestricted as well as types repository_suite_definition and - tool_dependency_definition. + tool_dependency_definition. """ encoded_repository_id = trans.security.encode_id(repository.id) repo = repository.hg_repo diff --git a/lib/tool_shed/test/functional/test_0440_deleting_dependency_definitions.py b/lib/tool_shed/test/functional/test_0440_deleting_dependency_definitions.py index f182b205da7..411512cf60f 100644 --- a/lib/tool_shed/test/functional/test_0440_deleting_dependency_definitions.py +++ b/lib/tool_shed/test/functional/test_0440_deleting_dependency_definitions.py @@ -22,6 +22,7 @@ bwa_tool_dependency_repository_long_description = "NT space mapping with BWA" ''' Simple repository dependencies: + 1. Create and populate column_maker_0440 so that it has an installable revision 0. 2. Create and populate convert_chars_0440 so that it has an installable revision 0. 3. Add a valid simple repository_dependencies.xml to convert_chars_0440 that points to the installable revision of column_maker_0440. @@ -29,6 +30,7 @@ Simple repository dependencies: 5. Delete repository_dependencies.xml from convert_chars_0440, and make sure convert_chars_0440 now has two installable revisions: 1 and 2 Complex repository dependencies: + 1. Create and populate bwa_package_0440 so that it has a valid tool dependency definition and an installable revision 0. 2. Create and populate bwa_base_0440 so that it has an installable revision 0. 3. Add a valid complex repository dependency tool_dependencies.xml to bwa_base_0440 that points to the installable revision 0 of bwa_package_0440. @@ -36,11 +38,13 @@ Complex repository dependencies: 5. Delete tool_dependencies.xml from bwa_base_0440, and make sure bwa_base_0440 now has two installable revisions: 1 and 2 Tool dependencies: + 1. Create and populate bwa_tool_dependency_0440 so that it has a valid tool dependency definition and an installable revision 0. 2. Delete tool_dependencies.xml from bwa_tool_dependency_0440, and make sure that bwa_tool_dependency_0440 still has a single installable revision 0. 3. Add the same tool_dependencies.xml file to bwa_tool_dependency_0440, and make sure that bwa_tool_dependency_0440 still has a single installable revision 0. + ''' diff --git a/lib/tool_shed/test/functional/test_1160_tool_help_images.py b/lib/tool_shed/test/functional/test_1160_tool_help_images.py index 5cdaf04d1e1..754cba287b8 100644 --- a/lib/tool_shed/test/functional/test_1160_tool_help_images.py +++ b/lib/tool_shed/test/functional/test_1160_tool_help_images.py @@ -11,13 +11,10 @@ repository_long_description = "Convert bed to gff" category_name = 'Test 0140 Tool Help Images' category_description = 'Test 0140 Tool Help Images' -''' -1) Create and populate the htseq_count_0140 repository. -2) Visit the manage_repository page, then the tool page, and look for the image string -similar to the following string where the encoded repository_id is previously determined: - -src="/repository/static/images//count_modes.png" -''' +# 1) Create and populate the htseq_count_0140 repository. +# 2) Visit the manage_repository page, then the tool page, and look for the image string +# similar to the following string where the encoded repository_id is previously determined: +# src="/repository/static/images//count_modes.png" class TestToolHelpImages(ShedTwillTestCase): diff --git a/lib/tool_shed/test/functional/test_1300_reset_all_metadata.py b/lib/tool_shed/test/functional/test_1300_reset_all_metadata.py index cf0f41617de..6dd5829ffc5 100644 --- a/lib/tool_shed/test/functional/test_1300_reset_all_metadata.py +++ b/lib/tool_shed/test/functional/test_1300_reset_all_metadata.py @@ -24,8 +24,7 @@ category_0060_name = 'Test 0060 Workflow Features' ''' This script will run in one of two possible ways: -1. Directly, by calling sh run_functional_tests.sh -toolshed test/tool_shed/functional/test_1300_reset_all_metadata.py. - -OR- +1. Directly, by calling sh run_functional_tests.sh -toolshed test/tool_shed/functional/test_1300_reset_all_metadata.py. -or- 2. After the previous test scripts have completed. In the first case, it is desirable to have the Galaxy database in a state that is as close as possible to the state it would diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index 21aedfbbead..747041ab96c 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -303,6 +303,7 @@ def get_tool_path_by_shed_tool_conf_filename(app, shed_tool_conf): def handle_email_alerts(app, host, repository, content_alert_str='', new_repo_alert=False, admin_only=False): """ There are 2 complementary features that enable a tool shed user to receive email notification: + 1. Within User Preferences, they can elect to receive email when the first (or first valid) change set is produced for a new repository. 2. When viewing or managing a repository, they can check the box labeled "Receive email alerts" @@ -310,18 +311,20 @@ def handle_email_alerts(app, host, repository, content_alert_str='', new_repo_al is available on a per-repository basis on the repository grid within the tool shed. There are currently 4 scenarios for sending email notification when a change is made to a repository: + 1. An admin user elects to receive email when the first change set is produced for a new repository from User Preferences. The change set does not have to include any valid content. This allows for the capture of inappropriate content being uploaded to new repositories. 2. A regular user elects to receive email when the first valid change set is produced for a new repository from User Preferences. This differs from 1 above in that the user will not receive email until a - change set tha tincludes valid content is produced. + change set that includes valid content is produced. 3. An admin user checks the "Receive email alerts" check box on the manage repository page. Since the user is an admin user, the email will include information about both HTML and image content that was included in the change set. 4. A regular user checks the "Receive email alerts" check box on the manage repository page. Since the user is not an admin user, the email will not include any information about both HTML and image content that was included in the change set. + """ sa_session = app.model.context.current repo = repository.hg_repo diff --git a/lib/tool_shed/webapp/controllers/repository_review.py b/lib/tool_shed/webapp/controllers/repository_review.py index fa694bc70ea..975571fcb54 100644 --- a/lib/tool_shed/webapp/controllers/repository_review.py +++ b/lib/tool_shed/webapp/controllers/repository_review.py @@ -389,6 +389,7 @@ class RepositoryReviewController(BaseUIController, ratings_util.ItemRatings): def manage_repositories_ready_for_review(self, trans, **kwd): """ A repository is ready to be reviewed if one of the following conditions is met: + 1) It contains no tools 2) It contains tools the tools_functionally_correct flag is set to True. This implies that the repository metadata revision was installed and tested by the Tool Shed's install and test framework. From 553ba52080ed6ae150c2f3d11bc5e5e556ff0cfc Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 11 Jan 2021 22:30:43 -0500 Subject: [PATCH 077/208] Fail sphinx document generation on warnings. --- .github/workflows/docs.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index f4445f16948..c94dcfef195 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -1,5 +1,7 @@ name: Build docs on: [push, pull_request] +env: + SPHINXOPTS: '-W' jobs: build: runs-on: ubuntu-latest From 99186b6294e86c6099846695a3e0b80f41ad1f9b Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 12 Jan 2021 00:43:22 -0500 Subject: [PATCH 078/208] Remove sphinx stub generated due to local, non-branch changes. --- doc/source/lib/galaxy.tool_util.deps.rst | 8 -------- 1 file changed, 8 deletions(-) diff --git a/doc/source/lib/galaxy.tool_util.deps.rst b/doc/source/lib/galaxy.tool_util.deps.rst index ee9da16f461..947dddb4f28 100644 --- a/doc/source/lib/galaxy.tool_util.deps.rst +++ b/doc/source/lib/galaxy.tool_util.deps.rst @@ -51,14 +51,6 @@ galaxy.tool\_util.deps.conda\_compat module :undoc-members: :show-inheritance: -galaxy.tool\_util.deps.conda\_init module ------------------------------------------ - -.. automodule:: galaxy.tool_util.deps.conda_init - :members: - :undoc-members: - :show-inheritance: - galaxy.tool\_util.deps.conda\_util module ----------------------------------------- From 45ed3178327c651ce9267838a2078c48ed775883 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 12 Jan 2021 07:58:23 -0500 Subject: [PATCH 079/208] Rework docs to -W more aggressively but also provide a development Makefile target. Development target uses 4 cores (I use 8 locally), shows all warnings about issues (instead of failing on first), and skips the viewcode step which isn't parallelized and doesn't really exercise anything the developer has useful control over or would need to see during doc tweaking. --- .github/workflows/docs.yaml | 2 -- Makefile | 4 ++-- doc/Makefile | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index c94dcfef195..f4445f16948 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -1,7 +1,5 @@ name: Build docs on: [push, pull_request] -env: - SPHINXOPTS: '-W' jobs: build: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 2e123d225b2..938f7bf63fd 100644 --- a/Makefile +++ b/Makefile @@ -30,8 +30,8 @@ docs: ## Generate HTML documentation. $(IN_VENV) $(MAKE) -C doc clean $(IN_VENV) $(MAKE) -C doc html -rebuild-docs: ## Re-generate HTML documentation without clean step. - $(IN_VENV) $(MAKE) -C doc html +docs-develop: ## Fast doc generation and more warnings (for development) + $(IN_VENV) GALAXY_DOCS_SKIP_VIEW_CODE=1 SPHINXOPTS='-j 4' $(MAKE) -C doc html setup-venv: if [ ! -f $(VENV)/bin/activate ]; then bash scripts/common_startup.sh --dev-wheels; fi diff --git a/doc/Makefile b/doc/Makefile index de5921267ad..2b37c840bb5 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -2,7 +2,7 @@ # # You can set these variables from the command line. -SPHINXOPTS ?= +SPHINXOPTS ?= -W SPHINXBUILD = sphinx-build PAPER = BUILDDIR = build From 955c6a8053a56889a20b821ea7b920897238c51a Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 12 Jan 2021 08:07:05 -0500 Subject: [PATCH 080/208] Index the user release notes when available by default. https://github.com/galaxyproject/galaxy/pull/11103#discussion_r555514754 --- doc/source/releases/index.rst | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/doc/source/releases/index.rst b/doc/source/releases/index.rst index 9126bc636ec..3d6157b8b48 100644 --- a/doc/source/releases/index.rst +++ b/doc/source/releases/index.rst @@ -4,12 +4,12 @@ Releases .. toctree:: :maxdepth: 1 - 20.09_announce - 20.05_announce - 20.01_announce - 19.09_announce - 19.05_announce - 19.01_announce + 20.09_announce_user + 20.05_announce_user + 20.01_announce_user + 19.09_announce_user + 19.05_announce_user + 19.01_announce_user 18.09_announce 18.05_announce 18.01_announce @@ -37,15 +37,3 @@ Releases 13.02_announce 13.01_announce older_releases - -User Announcements - -.. toctree:: - :maxdepth: 1 - - 20.09_announce_user - 20.05_announce_user - 20.01_announce_user - 19.09_announce_user - 19.05_announce_user - 19.01_announce_user From b8c11ab21a22b7cac4a32267b7b6e00a9858ab45 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 12 Jan 2021 08:41:30 -0500 Subject: [PATCH 081/208] Link in MQ docs stub. --- doc/source/admin/index.rst | 1 + doc/source/admin/mq.md | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/doc/source/admin/index.rst b/doc/source/admin/index.rst index 690774a1e66..79e79215658 100644 --- a/doc/source/admin/index.rst +++ b/doc/source/admin/index.rst @@ -18,6 +18,7 @@ This documentation is in the midst of being ported and unified based on resource jobs authentication tool_panel + mq dependency_resolvers conda_faq reports diff --git a/doc/source/admin/mq.md b/doc/source/admin/mq.md index cfe171a1564..07936d1ab69 100644 --- a/doc/source/admin/mq.md +++ b/doc/source/admin/mq.md @@ -1,6 +1,3 @@ -```eval_rst -:orphan: -``` # Message queue From ccdbc291601715b1088521bd9e2d65a180f3bcec Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 12 Jan 2021 09:35:19 -0500 Subject: [PATCH 082/208] Don't fail on sphinx warnings. --- doc/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Makefile b/doc/Makefile index 2b37c840bb5..de5921267ad 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -2,7 +2,7 @@ # # You can set these variables from the command line. -SPHINXOPTS ?= -W +SPHINXOPTS ?= SPHINXBUILD = sphinx-build PAPER = BUILDDIR = build From d69203e6985c4a517e79aab41b513bd92af1e172 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 12 Jan 2021 15:38:52 +0100 Subject: [PATCH 083/208] Fix converters response serialization (legacy) --- lib/galaxy/managers/datatypes.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/managers/datatypes.py b/lib/galaxy/managers/datatypes.py index 1512e699419..97c2bf895ed 100644 --- a/lib/galaxy/managers/datatypes.py +++ b/lib/galaxy/managers/datatypes.py @@ -189,14 +189,14 @@ def view_sniffers(datatypes_registry: Registry) -> List[str]: def view_converters(datatypes_registry: Registry) -> List[DatatypeConverter]: - converters: List[DatatypeConverter] = [] + converters = [] for (source_type, targets) in datatypes_registry.datatype_converters.items(): for target_type in targets: - converters.append(DatatypeConverter( - source=source_type, - target=target_type, - tool_id=targets[target_type].id, - )) + converters.append({ + 'source': source_type, + 'target': target_type, + 'tool_id': targets[target_type].id, + }) return converters From c518755d74e7c0d7f1b3e4a00e202d68cd1d564a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20L=C3=B3pez?= <46503462+davelopez@users.noreply.github.com> Date: Tue, 12 Jan 2021 15:55:45 +0100 Subject: [PATCH 084/208] Update lib/galaxy_test/api/test_datatypes.py Co-authored-by: Marius van den Beek --- lib/galaxy_test/api/test_datatypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy_test/api/test_datatypes.py b/lib/galaxy_test/api/test_datatypes.py index e310cf11bbe..3cc25f8eadf 100644 --- a/lib/galaxy_test/api/test_datatypes.py +++ b/lib/galaxy_test/api/test_datatypes.py @@ -84,7 +84,7 @@ class DatatypesApiTestCase(ApiTestCase): self._assert_status_code_is(response, 200) assert converters == len(response.json()) - def test_edamp_formats(self): + def test_edam_formats(self): response = self._get("datatypes/edam_formats") self._assert_status_code_is(response, 200) edam_formats = response.json() From 61abc3fcc26cf8cc389521bcc524c5a114926aa4 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 12 Jan 2021 15:57:56 +0100 Subject: [PATCH 085/208] Fix typing serialization --- lib/galaxy/managers/datatypes.py | 12 ++++++++++-- lib/galaxy/webapps/galaxy/api/datatypes.py | 6 +++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/managers/datatypes.py b/lib/galaxy/managers/datatypes.py index 97c2bf895ed..edce2737e17 100644 --- a/lib/galaxy/managers/datatypes.py +++ b/lib/galaxy/managers/datatypes.py @@ -10,6 +10,7 @@ from pydantic import ( Field, HttpUrl ) +from pydantic.tools import parse_obj_as from galaxy.datatypes.data import Data from galaxy.datatypes.registry import Registry @@ -126,6 +127,13 @@ class DatatypeConverter(BaseModel): ) +class DatatypeConverterList(BaseModel): + __root__: List[DatatypeConverter] = Field( + title='List of data type converters', + default=[] + ) + + def view_index( datatypes_registry: Registry, extension_only: Optional[bool] = True, @@ -188,7 +196,7 @@ def view_sniffers(datatypes_registry: Registry) -> List[str]: return rval -def view_converters(datatypes_registry: Registry) -> List[DatatypeConverter]: +def view_converters(datatypes_registry: Registry) -> DatatypeConverterList: converters = [] for (source_type, targets) in datatypes_registry.datatype_converters.items(): for target_type in targets: @@ -197,7 +205,7 @@ def view_converters(datatypes_registry: Registry) -> List[DatatypeConverter]: 'target': target_type, 'tool_id': targets[target_type].id, }) - return converters + return parse_obj_as(DatatypeConverterList, converters) def view_edam_formats(datatypes_registry: Registry) -> Dict[str, str]: diff --git a/lib/galaxy/webapps/galaxy/api/datatypes.py b/lib/galaxy/webapps/galaxy/api/datatypes.py index 820aadff2c7..178cf870b2c 100644 --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -19,7 +19,7 @@ from fastapi_utils.cbv import cbv from galaxy.app import UniverseApplication from galaxy.datatypes.registry import Registry from galaxy.managers.datatypes import ( - DatatypeConverter, + DatatypeConverterList, DatatypeDetails, DatatypesCombinedMap, DatatypesMap, @@ -115,10 +115,10 @@ class FastAPIDatatypes: @router.get( '/api/datatypes/converters', summary="Returns the list of all installed converters", - response_model=List[DatatypeConverter], + response_model=DatatypeConverterList, response_description="List of all datatype converters" ) - async def converters(self) -> List[DatatypeConverter]: + async def converters(self) -> DatatypeConverterList: """Gets the list of all installed converters.""" return view_converters(self.datatypes_registry) From a310047bd3ef12f3a6e25257c4bf14509b92b24f Mon Sep 17 00:00:00 2001 From: kxk302 Date: Tue, 12 Jan 2021 11:00:26 -0500 Subject: [PATCH 086/208] Update lib/galaxy/tools/parameters/basic.py Co-authored-by: Marius van den Beek --- lib/galaxy/tools/parameters/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index d2bab941599..b23fac67123 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1349,7 +1349,7 @@ class ColumnListParameter(SelectToolParameter): # Use representative dataset if a dataset collection is parsed if isinstance(dataset, trans.app.model.HistoryDatasetCollectionAssociation): dataset = dataset.to_hda_representative() - if not is_runtime_value(dataset) and not dataset.has_data(): + if is_runtime_value(dataset) or not dataset.has_data(): return True return False From 975a6e72c84a304f0d6c4b77397913f7df4f555e Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 12 Jan 2021 17:45:55 +0100 Subject: [PATCH 087/208] make example valid Co-authored-by: Marius van den Beek --- lib/galaxy/tool_util/xsd/galaxy.xsd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index c5bc7c47d1c..c2bd556c7a5 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2383,8 +2383,8 @@ The XML configuration is relatively trivial for sections: ```xml -
- +
+
``` From 07aa0fa271bf1b342c4d8ffca2e8293c16f3f974 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 12 Jan 2021 13:48:20 +0100 Subject: [PATCH 088/208] improve legal value check --- lib/galaxy/tools/parameters/basic.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 8d090ee158c..32dd8f1bbfd 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -928,17 +928,12 @@ class SelectToolParameter(ToolParameter): if isinstance(value, list): if not self.multiple: raise ParameterValueError("multiple values provided but parameter is not expecting multiple values", self.name, is_dynamic=self.is_dynamic) - rval = [] - for v in value: - if v in legal_values: - rval.append(v) - if len(rval) == 0: - for v in value: - if v in fallback_values: - rval.append(fallback_values[v]) - if len(rval) == 0: - raise ParameterValueError("an invalid option ({!r}) was selected (valid options: {})".format(v, ",".join(legal_values)), self.name, v, is_dynamic=self.is_dynamic) - return rval + if set(value).issubset(legal_values): + return value + elif set(value).issubset(set(fallback_values.keys())): + return [fallback_values[v] for v in value] + else: + raise ParameterValueError("invalid options ({!r}) were selected (valid options: {})".format(",".join(set(value) - set(legal_values)), ",".join(legal_values)), self.name, is_dynamic=self.is_dynamic) else: value_is_none = (value == "None" and "None" not in legal_values) if value_is_none or not value: From 5f67c2075b88296af1a2924ff96262e014a6b15b Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 12 Jan 2021 19:27:07 +0100 Subject: [PATCH 089/208] fix name and value column in select_from_dataset.xml --- test/functional/tools/select_from_dataset.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/functional/tools/select_from_dataset.xml b/test/functional/tools/select_from_dataset.xml index 1cb117129ef..b5e949025cb 100644 --- a/test/functional/tools/select_from_dataset.xml +++ b/test/functional/tools/select_from_dataset.xml @@ -9,7 +9,7 @@ echo select_collection '$select_collection' >> '$output' - + @@ -17,7 +17,7 @@ echo select_collection '$select_collection' >> '$output' - + @@ -25,7 +25,7 @@ echo select_collection '$select_collection' >> '$output' - + From d90b4d24a29cba1d8517577697921dab22734498 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 12 Jan 2021 13:46:11 -0500 Subject: [PATCH 090/208] Refactor tours package organization --- lib/galaxy/app.py | 4 +- lib/galaxy/tours/__init__.py | 103 ++---------------- lib/galaxy/tours/_impl.py | 100 +++++++++++++++++ .../tours/{abc/__init__.py => _interface.py} | 2 +- lib/galaxy/tours/{schema.py => _schema.py} | 0 lib/galaxy/webapps/galaxy/api/__init__.py | 5 - lib/galaxy/webapps/galaxy/api/tours.py | 10 +- 7 files changed, 117 insertions(+), 107 deletions(-) create mode 100644 lib/galaxy/tours/_impl.py rename lib/galaxy/tours/{abc/__init__.py => _interface.py} (92%) rename lib/galaxy/tours/{schema.py => _schema.py} (100%) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 97c6b971a7b..d773de8a734 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -43,7 +43,7 @@ from galaxy.tools.cache import ( from galaxy.tools.data_manager.manager import DataManagers from galaxy.tools.error_reports import ErrorReports from galaxy.tools.special_tools import load_lib_tools -from galaxy.tours import ToursRegistry +from galaxy.tours import build_tours_registry from galaxy.util import ( ExecutionTimer, heartbeat, @@ -173,7 +173,7 @@ class UniverseApplication(config.ConfiguresGalaxyMixin): directories_setting=self.config.visualization_plugins_directory, template_cache_dir=self.config.template_cache_path) # Tours registry - self.tour_registry = ToursRegistry(self.config.tour_config_dir) + self.tour_registry = build_tours_registry(self.config.tour_config_dir) # Webhooks registry self.webhooks_registry = WebhooksRegistry(self.config.webhooks_dir) # Load security policy. diff --git a/lib/galaxy/tours/__init__.py b/lib/galaxy/tours/__init__.py index 478c0fb05ce..060f7f161a9 100644 --- a/lib/galaxy/tours/__init__.py +++ b/lib/galaxy/tours/__init__.py @@ -1,96 +1,7 @@ -""" -This module manages loading/etc of Galaxy interactive tours. -""" -import logging -import os - -import yaml -from pydantic import parse_obj_as - -from galaxy import util -from galaxy.tours import abc -from galaxy.tours.schema import TourList - - -log = logging.getLogger(__name__) - - -def load_steps(contents_dict): - # Some of this can be done on the clientside. Maybe even should? - title_default = contents_dict.get('title_default', None) - for step in contents_dict['steps']: - if 'intro' in step: - step['content'] = step.pop('intro') - if 'position' in step: - step['placement'] = step.pop('position') - if 'element' not in step: - step['orphan'] = True - if title_default and 'title' not in step: - step['title'] = title_default - - -@abc.ToursRegistry.register -class ToursRegistry: - def __init__(self, tour_directories): - self.tour_directories = util.config_directories_from_setting(tour_directories) - self.load_tours() - - def tours_by_id_with_description(self): - tours = [] - for k in self.tours.keys(): - tourdata = { - 'id': k, - 'description': self.tours[k].get('description'), - 'name': self.tours[k].get('name'), - 'tags': self.tours[k].get('tags') - } - tours.append(tourdata) - return parse_obj_as(TourList, tours) - - def load_tour(self, tour_id): - for tour_dir in self.tour_directories: - tour_path = os.path.join(tour_dir, tour_id + ".yaml") - if not os.path.exists(tour_path): - tour_path = os.path.join(tour_dir, tour_id + ".yml") - if os.path.exists(tour_path): - return self._load_tour_from_path(tour_path) - - def load_tours(self): - self.tours = {} - for tour_dir in self.tour_directories: - for filename in os.listdir(tour_dir): - if self._is_yaml(filename): - self._load_tour_from_path(os.path.join(tour_dir, filename)) - return self.tours_by_id_with_description() - - def reload_tour(self, path): - # We may safely assume that the path is within the tour directory - filename = os.path.basename(path) - if self._is_yaml(filename): - self._load_tour_from_path(path) - - def tour_contents(self, tour_id): - # Extra format translation could happen here (like the previous intro_to_tour) - # For now just return the loaded contents. - return self.tours.get(tour_id, None) - - def _is_yaml(self, filename): - return filename.endswith('.yaml') or filename.endswith('.yml') - - def _load_tour_from_path(self, tour_path): - filename = os.path.basename(tour_path) - tour_id = os.path.splitext(filename)[0] - try: - with open(tour_path) as handle: - tour = yaml.safe_load(handle) - load_steps(tour) - self.tours[tour_id] = tour - log.info("Loaded tour '%s'" % tour_id) - return tour - except OSError: - log.exception("Tour '%s' could not be loaded, error reading file.", tour_id) - except yaml.error.YAMLError: - log.exception("Tour '%s' could not be loaded, error within file. Please check your yaml syntax.", tour_id) - except TypeError: - log.exception("Tour '%s' could not be loaded, error within file. Possibly spacing related. Please check your yaml syntax.", tour_id) - return None +from ._impl import build_tours_registry # noqa +from ._interface import ToursRegistry # noqa +from ._schema import Tour # noqa +from ._schema import TourCore # noqa +from ._schema import TourStep # noqa +from ._schema import TourDetails # noqa +from ._schema import TourList # noqa diff --git a/lib/galaxy/tours/_impl.py b/lib/galaxy/tours/_impl.py new file mode 100644 index 00000000000..a88d5fff479 --- /dev/null +++ b/lib/galaxy/tours/_impl.py @@ -0,0 +1,100 @@ +""" +This module manages loading/etc of Galaxy interactive tours. +""" +import logging +import os + +import yaml +from pydantic import parse_obj_as + +from galaxy import util +from ._interface import ToursRegistry +from ._schema import TourList + + +log = logging.getLogger(__name__) + + +def build_tours_registry(tour_directories: str) -> ToursRegistry: + return ToursRegistryImpl(tour_directories) + + +def load_steps(contents_dict): + # Some of this can be done on the clientside. Maybe even should? + title_default = contents_dict.get('title_default', None) + for step in contents_dict['steps']: + if 'intro' in step: + step['content'] = step.pop('intro') + if 'position' in step: + step['placement'] = step.pop('position') + if 'element' not in step: + step['orphan'] = True + if title_default and 'title' not in step: + step['title'] = title_default + + +@ToursRegistry.register +class ToursRegistryImpl: + def __init__(self, tour_directories): + self.tour_directories = util.config_directories_from_setting(tour_directories) + self.load_tours() + + def tours_by_id_with_description(self): + tours = [] + for k in self.tours.keys(): + tourdata = { + 'id': k, + 'description': self.tours[k].get('description'), + 'name': self.tours[k].get('name'), + 'tags': self.tours[k].get('tags') + } + tours.append(tourdata) + return parse_obj_as(TourList, tours) + + def load_tour(self, tour_id): + for tour_dir in self.tour_directories: + tour_path = os.path.join(tour_dir, tour_id + ".yaml") + if not os.path.exists(tour_path): + tour_path = os.path.join(tour_dir, tour_id + ".yml") + if os.path.exists(tour_path): + return self._load_tour_from_path(tour_path) + + def load_tours(self): + self.tours = {} + for tour_dir in self.tour_directories: + for filename in os.listdir(tour_dir): + if self._is_yaml(filename): + self._load_tour_from_path(os.path.join(tour_dir, filename)) + return self.tours_by_id_with_description() + + def reload_tour(self, path): + # We may safely assume that the path is within the tour directory + filename = os.path.basename(path) + if self._is_yaml(filename): + self._load_tour_from_path(path) + + def tour_contents(self, tour_id): + # Extra format translation could happen here (like the previous intro_to_tour) + # For now just return the loaded contents. + return self.tours.get(tour_id, None) + + def _is_yaml(self, filename): + return filename.endswith('.yaml') or filename.endswith('.yml') + + def _load_tour_from_path(self, tour_path): + filename = os.path.basename(tour_path) + tour_id = os.path.splitext(filename)[0] + try: + with open(tour_path) as handle: + tour = yaml.safe_load(handle) + load_steps(tour) + self.tours[tour_id] = tour + log.info("Loaded tour '%s'" % tour_id) + return tour + except OSError: + log.exception("Tour '%s' could not be loaded, error reading file.", tour_id) + except yaml.error.YAMLError: + log.exception("Tour '%s' could not be loaded, error within file. Please check your yaml syntax.", tour_id) + except TypeError: + log.exception("Tour '%s' could not be loaded, error within file. Possibly spacing related. Please check your yaml syntax.", tour_id) + return None diff --git a/lib/galaxy/tours/abc/__init__.py b/lib/galaxy/tours/_interface.py similarity index 92% rename from lib/galaxy/tours/abc/__init__.py rename to lib/galaxy/tours/_interface.py index ba0b1f06c5d..82fb223491f 100644 --- a/lib/galaxy/tours/abc/__init__.py +++ b/lib/galaxy/tours/_interface.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod -from galaxy.tours.schema import ( +from ._schema import ( TourDetails, TourList, ) diff --git a/lib/galaxy/tours/schema.py b/lib/galaxy/tours/_schema.py similarity index 100% rename from lib/galaxy/tours/schema.py rename to lib/galaxy/tours/_schema.py diff --git a/lib/galaxy/webapps/galaxy/api/__init__.py b/lib/galaxy/webapps/galaxy/api/__init__.py index 9f0dd36b6d7..4665a075b70 100644 --- a/lib/galaxy/webapps/galaxy/api/__init__.py +++ b/lib/galaxy/webapps/galaxy/api/__init__.py @@ -24,7 +24,6 @@ from galaxy.managers.jobs import JobManager from galaxy.managers.session import GalaxySessionManager from galaxy.managers.users import UserManager from galaxy.model import User -from galaxy.tours.abc import ToursRegistry from galaxy.web.framework.decorators import require_admin_message from galaxy.work.context import SessionRequestContext @@ -86,7 +85,3 @@ def get_admin_user(trans: SessionRequestContext = Depends(get_trans)): if not trans.user_is_admin: raise AdminRequiredException(require_admin_message(trans.app.config, trans.user)) return trans.user - - -def get_tours_registry(app: UniverseApplication = Depends(get_app)) -> ToursRegistry: - return app.tour_registry diff --git a/lib/galaxy/webapps/galaxy/api/tours.py b/lib/galaxy/webapps/galaxy/api/tours.py index dddcca3a364..843530f8b59 100644 --- a/lib/galaxy/webapps/galaxy/api/tours.py +++ b/lib/galaxy/webapps/galaxy/api/tours.py @@ -7,10 +7,10 @@ from fastapi import Depends from fastapi_utils.cbv import cbv from fastapi_utils.inferring_router import InferringRouter as APIRouter -from galaxy.tours.abc import ToursRegistry -from galaxy.tours.schema import ( +from galaxy.tours import ( TourDetails, TourList, + ToursRegistry, ) from galaxy.web import ( expose_api_anonymous_and_sessionless, @@ -20,7 +20,7 @@ from galaxy.web import ( from galaxy.webapps.base.controller import BaseAPIController from . import ( get_admin_user, - get_tours_registry, + get_app, ) log = logging.getLogger(__name__) @@ -29,6 +29,10 @@ log = logging.getLogger(__name__) router = APIRouter(tags=['tours']) +def get_tours_registry(app=Depends(get_app)) -> ToursRegistry: + return app.tour_registry + + @cbv(router) class FastAPITours: registry: ToursRegistry = Depends(get_tours_registry) From b21f657761a1c6e303a4d255276532564438d1fd Mon Sep 17 00:00:00 2001 From: Dave B Date: Tue, 12 Jan 2021 14:13:16 -0500 Subject: [PATCH 091/208] Update lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py Co-authored-by: Marius van den Beek --- .../tool_shed/galaxy_install/tools/tool_panel_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py b/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py index ff604fd6d19..3913134499b 100644 --- a/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -67,7 +67,8 @@ class ToolPanelManager: # Break out of the config_elems loop back to the elem_list loop. break # If we reach this point, no section was found. Create one with contents. - config_elems.append(elem_entry) + else: + config_elems.append(elem_entry) # This is not a section, but a tool or label. No need to search for matching sections, just add it. else: config_elems.append(elem_entry) From dcaa04b69073325c53ad98038f9aebcf7e802b5c Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 12 Jan 2021 15:31:55 -0500 Subject: [PATCH 092/208] Refactor tours module implemantation --- lib/galaxy/tours/_impl.py | 89 +++++++++++++++----------- lib/galaxy/tours/_interface.py | 6 +- lib/galaxy/webapps/galaxy/api/tours.py | 4 +- 3 files changed, 57 insertions(+), 42 deletions(-) diff --git a/lib/galaxy/tours/_impl.py b/lib/galaxy/tours/_impl.py index a88d5fff479..58f1b07d094 100644 --- a/lib/galaxy/tours/_impl.py +++ b/lib/galaxy/tours/_impl.py @@ -7,7 +7,7 @@ import os import yaml from pydantic import parse_obj_as -from galaxy import util +from galaxy.util import config_directories_from_setting from ._interface import ToursRegistry from ._schema import TourList @@ -19,9 +19,9 @@ def build_tours_registry(tour_directories: str) -> ToursRegistry: return ToursRegistryImpl(tour_directories) -def load_steps(contents_dict): +def load_tour_steps(contents_dict): # Some of this can be done on the clientside. Maybe even should? - title_default = contents_dict.get('title_default', None) + title_default = contents_dict.get('title_default') for step in contents_dict['steps']: if 'intro' in step: step['content'] = step.pop('intro') @@ -35,66 +35,81 @@ def load_steps(contents_dict): @ToursRegistry.register class ToursRegistryImpl: - def __init__(self, tour_directories): - self.tour_directories = util.config_directories_from_setting(tour_directories) - self.load_tours() - def tours_by_id_with_description(self): + def __init__(self, tour_directories): + self.tour_directories = config_directories_from_setting(tour_directories) + self._extensions = ('.yml', '.yaml') + self._load_tours() + + def get_tours(self): + """Return list of tours.""" tours = [] for k in self.tours.keys(): tourdata = { 'id': k, - 'description': self.tours[k].get('description'), 'name': self.tours[k].get('name'), + 'description': self.tours[k].get('description'), 'tags': self.tours[k].get('tags') } tours.append(tourdata) return parse_obj_as(TourList, tours) - def load_tour(self, tour_id): - for tour_dir in self.tour_directories: - tour_path = os.path.join(tour_dir, tour_id + ".yaml") - if not os.path.exists(tour_path): - tour_path = os.path.join(tour_dir, tour_id + ".yml") - if os.path.exists(tour_path): - return self._load_tour_from_path(tour_path) + def tour_contents(self, tour_id): + """Return tour contents.""" + # Extra format translation could happen here (like the previous intro_to_tour) + # For now just return the loaded contents. + return self.tours.get(tour_id) - def load_tours(self): - self.tours = {} - for tour_dir in self.tour_directories: - for filename in os.listdir(tour_dir): - if self._is_yaml(filename): - self._load_tour_from_path(os.path.join(tour_dir, filename)) - return self.tours_by_id_with_description() + def load_tour(self, tour_id): + """Reload tour and return its contents.""" + tour_path = self._get_path_from_tour_id(tour_id) + self._load_tour_from_path(tour_path) + return self.tours.get(tour_id) def reload_tour(self, path): + """Reload tour.""" # We may safely assume that the path is within the tour directory filename = os.path.basename(path) if self._is_yaml(filename): self._load_tour_from_path(path) - def tour_contents(self, tour_id): - # Extra format translation could happen here (like the previous intro_to_tour) - # For now just return the loaded contents. - return self.tours.get(tour_id, None) + def _load_tours(self): + self.tours = {} + for tour_dir in self.tour_directories: + for filename in os.listdir(tour_dir): + if self._is_yaml(filename): + tour_path = os.path.join(tour_dir, filename) + self._load_tour_from_path(tour_path) def _is_yaml(self, filename): - return filename.endswith('.yaml') or filename.endswith('.yml') + for ext in self._extensions: + if filename.endswith(ext): + return True def _load_tour_from_path(self, tour_path): - filename = os.path.basename(tour_path) - tour_id = os.path.splitext(filename)[0] + tour_id = self._get_tour_id_from_path(tour_path) try: - with open(tour_path) as handle: - tour = yaml.safe_load(handle) - load_steps(tour) + with open(tour_path) as f: + tour = yaml.safe_load(f) + load_tour_steps(tour) self.tours[tour_id] = tour log.info("Loaded tour '%s'" % tour_id) - return tour except OSError: - log.exception("Tour '%s' could not be loaded, error reading file.", tour_id) + log.exception("Tour '%s' could not be loaded, error reading file." % tour_id) except yaml.error.YAMLError: - log.exception("Tour '%s' could not be loaded, error within file. Please check your yaml syntax.", tour_id) + log.exception("Tour '%s' could not be loaded, error within file." + " Please check your yaml syntax." % tour_id) except TypeError: - log.exception("Tour '%s' could not be loaded, error within file. Possibly spacing related. Please check your yaml syntax.", tour_id) - return None + log.exception("Tour '%s' could not be loaded, error within file." + " Possibly spacing related. Please check your yaml syntax." % tour_id) + + def _get_tour_id_from_path(self, tour_path): + filename = os.path.basename(tour_path) + return os.path.splitext(filename)[0] + + def _get_path_from_tour_id(self, tour_id): + for tour_dir in self.tour_directories: + for ext in self._extensions: + tour_path = os.path.join(tour_dir, tour_id + ext) + if os.path.exists(tour_path): + return tour_path diff --git a/lib/galaxy/tours/_interface.py b/lib/galaxy/tours/_interface.py index 82fb223491f..33cd2f76168 100644 --- a/lib/galaxy/tours/_interface.py +++ b/lib/galaxy/tours/_interface.py @@ -9,13 +9,13 @@ from ._schema import ( class ToursRegistry(ABC): @abstractmethod - def tours_by_id_with_description(self) -> TourList: + def get_tours(self) -> TourList: """Return list of tours.""" @abstractmethod def tour_contents(self, tour_id: str) -> TourDetails: - """Return tour details.""" + """Return tour contents.""" @abstractmethod def load_tour(self, tour_id: str) -> TourDetails: - """Reload tour and return tour details.""" + """Reload tour and return its contents.""" diff --git a/lib/galaxy/webapps/galaxy/api/tours.py b/lib/galaxy/webapps/galaxy/api/tours.py index 843530f8b59..fa0b126642c 100644 --- a/lib/galaxy/webapps/galaxy/api/tours.py +++ b/lib/galaxy/webapps/galaxy/api/tours.py @@ -40,7 +40,7 @@ class FastAPITours: @router.get('/api/tours') def index(self) -> TourList: """Return list of available tours.""" - return self.registry.tours_by_id_with_description() + return self.registry.get_tours() @router.get('/api/tours/{tour_id}') def show(self, tour_id: str) -> TourDetails: @@ -64,7 +64,7 @@ class ToursController(BaseAPIController): *GET /api/tours/ Displays available tours """ - return self.app.tour_registry.tours_by_id_with_description() + return self.app.tour_registry.get_tours() @expose_api_anonymous_and_sessionless def show(self, trans, tour_id, **kwd): From 6246cf94a4a40f8e5a00b5435aa29b930d708528 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 12 Jan 2021 14:48:31 -0500 Subject: [PATCH 093/208] Cleanup some docstring comments. - Fix spelling. - Drop a way pre-jmchilton deprecation notice. --- lib/galaxy/managers/context.py | 6 +++--- lib/galaxy/webapps/base/webapp.py | 3 --- lib/galaxy/work/context.py | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/managers/context.py b/lib/galaxy/managers/context.py index d927316571e..0e5c8789e2e 100644 --- a/lib/galaxy/managers/context.py +++ b/lib/galaxy/managers/context.py @@ -10,7 +10,7 @@ from galaxy.util import bunch class ProvidesAppContext: - """ For transaction-like objects to provide Galaxy convience layer for + """ For transaction-like objects to provide Galaxy convenience layer for database and event handling. Mixed in class must provide `app` property. @@ -100,7 +100,7 @@ class ProvidesAppContext: class ProvidesUserContext: - """ For transaction-like objects to provide Galaxy convience layer for + """ For transaction-like objects to provide Galaxy convenience layer for reasoning about users. Mixed in class must provide `user` and `app` @@ -160,7 +160,7 @@ class ProvidesUserContext: class ProvidesHistoryContext: - """ For transaction-like objects to provide Galaxy convience layer for + """ For transaction-like objects to provide Galaxy convenience layer for reasoning about histories. Mixed in class must provide `user`, `history`, and `app` diff --git a/lib/galaxy/webapps/base/webapp.py b/lib/galaxy/webapps/base/webapp.py index a9851d41ba8..97cf8fc99d2 100644 --- a/lib/galaxy/webapps/base/webapp.py +++ b/lib/galaxy/webapps/base/webapp.py @@ -430,9 +430,6 @@ class GalaxyWebTransaction(base.DefaultWebTransaction, """ Ensure that a valid Galaxy session exists and is available as trans.session (part of initialization) - - Support for universe_session and universe_user cookies has been - removed as of 31 Oct 2008. """ # Try to load an existing session secure_id = self.get_cookie(name=session_cookie) diff --git a/lib/galaxy/work/context.py b/lib/galaxy/work/context.py index 873d714301f..776b74b0601 100644 --- a/lib/galaxy/work/context.py +++ b/lib/galaxy/work/context.py @@ -8,7 +8,7 @@ from galaxy.managers.context import ( class WorkRequestContext(ProvidesAppContext, ProvidesUserContext, ProvidesHistoryContext): """ Stripped down implementation of Galaxy web transaction god object for work request handling outside of web threads - uses mix-ins shared with - GalaxyWebTransaction to provide app, user, and history context convience + GalaxyWebTransaction to provide app, user, and history context convenience methods - but nothing related to HTTP handling, mako views, etc.... Things that only need app shouldn't be consuming trans - but there is a From 8f3e5921e99f87598b0ab69ec387183d6621d087 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 12 Jan 2021 19:51:36 -0500 Subject: [PATCH 094/208] Work around limitations of mypy --- lib/galaxy/tours/_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tours/_impl.py b/lib/galaxy/tours/_impl.py index 58f1b07d094..be413f9d498 100644 --- a/lib/galaxy/tours/_impl.py +++ b/lib/galaxy/tours/_impl.py @@ -15,7 +15,7 @@ from ._schema import TourList log = logging.getLogger(__name__) -def build_tours_registry(tour_directories: str) -> ToursRegistry: +def build_tours_registry(tour_directories: str): return ToursRegistryImpl(tour_directories) From 1341c879dd771bae0a1f5a91c3c1118670020778 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 12 Jan 2021 15:09:18 -0500 Subject: [PATCH 095/208] Establish a best practice for trans type annotations. Provide justification and information in docstrings also. --- lib/galaxy/managers/context.py | 58 +++++++++++++++++-- lib/galaxy/managers/roles.py | 7 ++- lib/galaxy/model/__init__.py | 2 + lib/galaxy/webapps/base/webapp.py | 10 ++-- lib/galaxy/webapps/galaxy/api/jobs.py | 36 ++++++------ lib/galaxy/webapps/galaxy/api/roles.py | 22 ++++--- lib/galaxy/work/context.py | 11 ++-- test/integration/test_workflow_refactoring.py | 6 +- test/unit/test_galaxy_transactions.py | 6 +- 9 files changed, 108 insertions(+), 50 deletions(-) diff --git a/lib/galaxy/managers/context.py b/lib/galaxy/managers/context.py index 0e5c8789e2e..e4ed9cfbc0a 100644 --- a/lib/galaxy/managers/context.py +++ b/lib/galaxy/managers/context.py @@ -1,6 +1,40 @@ +"""Interfaces/mixins for transaction-like objects. + +These objects describe the context around a unit of work. This unit of work +is very broad and can be anything from the response to a web request, the +scheduling of a workflow, the reloading the toolbox, etc.. Traditionally, +Galaxy has simply passed around a GalaxyWebTransaction object through +all layers and large components of the Galaxy app. Having random backend +components define explicit dependencies on this however is inappropriate +because Galaxy may not be used in all sort of non-web contexts. The future +use of message queues and web sockets as well as the decomposition of the +backend into packages only further make this heavy reliance on +GalaxyWebTransaction inappropriate. + +A better approach is for components to annotate their reliance on much +narrower, typed views of the GalaxyWebTransaction. This allows explicit +declaration of what is being required in the context of a method or class +and allows the Python type system to ensure the transaction supplied to +the method is appropriate for the context. For instance, an effective +use of the type system in this way can prevent the backend work context +used to schedule workflow from being supplied to a method that requires +an older-style WSGI web response object. + +There are various levels of transactions defined in this file - these +include :class:`galaxy.managers.context.ProvidesAppContext`, +:class:`galaxy.managers.context.ProvidesUserContext`, +and :class:`galaxy.managers.context.ProvidesHistoryContext`. Methods +should annotate their dependency on the narrowest context they require. +A method that requires a user but not a history should declare its +``trans`` argument as requiring type :class:`galaxy.managers.context.ProvidesUserContext`. """ -Mixins for transaction-like objects. -""" +# TODO: Refactor this class so that galaxy.managers depends on a package +# containing this. +# TODO: Provide different classes for real users and potentially bootstrapped +# users so the framework can provide consistent web exceptions for incorrect +# user being used in that context and so that the type system can provide +# more checks against this issue. +import abc import string from json import dumps @@ -16,6 +50,14 @@ class ProvidesAppContext: Mixed in class must provide `app` property. """ + @abc.abstractproperty + def app(self): + """Provide access to the app object.""" + + @property + def security(self): + return self.app.security + def log_action(self, user=None, action=None, context=None, params=None): """ Application-level logging of user actions. @@ -99,7 +141,7 @@ class ProvidesAppContext: return self.app.install_model -class ProvidesUserContext: +class ProvidesUserContext(ProvidesAppContext): """ For transaction-like objects to provide Galaxy convenience layer for reasoning about users. @@ -107,6 +149,10 @@ class ProvidesUserContext: properties. """ + @abc.abstractproperty + def user(self): + """Provide access to the user object.""" + @property def anonymous(self): return self.user is None @@ -159,7 +205,7 @@ class ProvidesUserContext: return path -class ProvidesHistoryContext: +class ProvidesHistoryContext(ProvidesUserContext): """ For transaction-like objects to provide Galaxy convenience layer for reasoning about histories. @@ -167,6 +213,10 @@ class ProvidesHistoryContext: properties. """ + @abc.abstractproperty + def history(self): + """Provide access to the user's current history model object.""" + def db_dataset_for(self, dbkey): """ Returns the db_file dataset associated/needed by `dataset`, or `None`. diff --git a/lib/galaxy/managers/roles.py b/lib/galaxy/managers/roles.py index d1d709be029..6eb050d2450 100644 --- a/lib/galaxy/managers/roles.py +++ b/lib/galaxy/managers/roles.py @@ -15,6 +15,7 @@ import galaxy.exceptions from galaxy import model from galaxy.exceptions import RequestParameterInvalidException from galaxy.managers import base +from galaxy.managers.context import ProvidesUserContext from galaxy.schema.fields import EncodedDatabaseIdField from galaxy.util import unicodify @@ -54,7 +55,7 @@ class RoleManager(base.ModelManager): def __init__(self, app): super().__init__(app) - def get(self, trans, decoded_role_id): + def get(self, trans: ProvidesUserContext, decoded_role_id): """ Method loads the role from the DB based on the given role id. @@ -81,14 +82,14 @@ class RoleManager(base.ModelManager): return role - def list_displayable_roles(self, trans): + def list_displayable_roles(self, trans: ProvidesUserContext): roles = [] for role in trans.sa_session.query(trans.app.model.Role).filter(trans.app.model.Role.table.c.deleted == false()): if trans.user_is_admin or trans.app.security_agent.ok_to_display(trans.user, role): roles.append(role) return roles - def create(self, trans, role_definition_model): + def create_role(self, trans: ProvidesUserContext, role_definition_model) -> model.Role: name = role_definition_model.name description = role_definition_model.description user_ids = role_definition_model.user_ids or [] diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index c83769989f0..475c50b77eb 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -101,6 +101,8 @@ else: class RepresentById(_HasTable): + id: int + def __repr__(self): try: r = ''.format(self.__class__.__name__, cached_id(self), hex(id(self))) diff --git a/lib/galaxy/webapps/base/webapp.py b/lib/galaxy/webapps/base/webapp.py index 97cf8fc99d2..269d10135f3 100644 --- a/lib/galaxy/webapps/base/webapp.py +++ b/lib/galaxy/webapps/base/webapp.py @@ -177,17 +177,15 @@ class WebApplication(base.WebApplication): return T(app) -class GalaxyWebTransaction(base.DefaultWebTransaction, - context.ProvidesAppContext, context.ProvidesUserContext, context.ProvidesHistoryContext): +class GalaxyWebTransaction(base.DefaultWebTransaction, context.ProvidesHistoryContext): """ Encapsulates web transaction specific state for the Galaxy application (specifically the user's "cookie" session and history) """ def __init__(self, environ, app, webapp, session_cookie=None): - self.app = app + self._app = app self.webapp = webapp - self.security = webapp.security self.user_manager = UserManager(app) self.session_manager = GalaxySessionManager(app.model) base.DefaultWebTransaction.__init__(self, environ) @@ -264,6 +262,10 @@ class GalaxyWebTransaction(base.DefaultWebTransaction, self.sa_session.add(self.galaxy_session) self.sa_session.flush() + @property + def app(self): + return self._app + def setup_i18n(self): locales = [] if 'HTTP_ACCEPT_LANGUAGE' in self.environ: diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index a39a65cc130..a6996f9d88c 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -21,6 +21,7 @@ from galaxy import ( ) from galaxy.app import UniverseApplication from galaxy.managers import hdas +from galaxy.managers.context import ProvidesHistoryContext, ProvidesUserContext from galaxy.managers.jobs import ( JobLock, JobManager, @@ -41,7 +42,6 @@ from galaxy.webapps.base.controller import ( UsesVisualizationMixin ) from galaxy.work.context import ( - SessionRequestContext, WorkRequestContext, ) from . import ( @@ -70,7 +70,7 @@ class FastAPIJobs: hda_manager: hdas.HDAManager = Depends(get_hda_manager) @router.get("/api/job/{id}") - def show(self, id: EncodedDatabaseIdField, trans: SessionRequestContext = Depends(get_trans), full: typing.Optional[bool] = False) -> typing.Dict: + def show(self, id: EncodedDatabaseIdField, trans: ProvidesUserContext = Depends(get_trans), full: typing.Optional[bool] = False) -> typing.Dict: """ Return dictionary containing description of job data @@ -92,7 +92,7 @@ class JobController(BaseAPIController, UsesVisualizationMixin): self.hda_manager = hdas.HDAManager(app) @expose_api - def index(self, trans, **kwd): + def index(self, trans: ProvidesUserContext, **kwd): """ GET /api/jobs @@ -173,7 +173,7 @@ class JobController(BaseAPIController, UsesVisualizationMixin): return out @expose_api_anonymous - def show(self, trans, id, **kwd): + def show(self, trans: ProvidesUserContext, id, **kwd): """ show( trans, id ) * GET /api/jobs/{id}: @@ -193,7 +193,7 @@ class JobController(BaseAPIController, UsesVisualizationMixin): return view_show_job(trans, job, full_output) @expose_api - def common_problems(self, trans, id, **kwd): + def common_problems(self, trans: ProvidesUserContext, id, **kwd): """ * GET /api/jobs/{id}/common_problems check inputs and job for common potential problems to aid in error reporting @@ -219,7 +219,7 @@ class JobController(BaseAPIController, UsesVisualizationMixin): return {"has_empty_inputs": has_empty_inputs, "has_duplicate_inputs": has_duplicate_inputs} @expose_api - def inputs(self, trans, id, **kwd): + def inputs(self, trans: ProvidesUserContext, id, **kwd): """ GET /api/jobs/{id}/inputs @@ -235,7 +235,7 @@ class JobController(BaseAPIController, UsesVisualizationMixin): return self.__dictify_associations(trans, job.input_datasets, job.input_library_datasets) @expose_api - def outputs(self, trans, id, **kwd): + def outputs(self, trans: ProvidesUserContext, id, **kwd): """ outputs( trans, id ) * GET /api/jobs/{id}/outputs @@ -251,7 +251,7 @@ class JobController(BaseAPIController, UsesVisualizationMixin): return self.__dictify_associations(trans, job.output_datasets, job.output_library_datasets) @expose_api - def delete(self, trans, id, **kwd): + def delete(self, trans: ProvidesUserContext, id, **kwd): """ delete( trans, id ) * Delete /api/jobs/{id} @@ -268,7 +268,7 @@ class JobController(BaseAPIController, UsesVisualizationMixin): return self.job_manager.stop(job, message=message) @expose_api - def resume(self, trans, id, **kwd): + def resume(self, trans: ProvidesUserContext, id, **kwd): """ * PUT /api/jobs/{id}/resume Resumes a paused job @@ -289,7 +289,7 @@ class JobController(BaseAPIController, UsesVisualizationMixin): return self.__dictify_associations(trans, job.output_datasets, job.output_library_datasets) @expose_api_anonymous - def metrics(self, trans, **kwd): + def metrics(self, trans: ProvidesUserContext, **kwd): """ * GET /api/jobs/{job_id}/metrics * GET /api/datasets/{dataset_id}/metrics @@ -314,7 +314,7 @@ class JobController(BaseAPIController, UsesVisualizationMixin): @require_admin @expose_api - def destination_params(self, trans, **kwd): + def destination_params(self, trans: ProvidesUserContext, **kwd): """ * GET /api/jobs/{job_id}/destination_params Return destination parameters for specified job. @@ -329,7 +329,7 @@ class JobController(BaseAPIController, UsesVisualizationMixin): return summarize_destination_params(trans, job) @expose_api_anonymous - def parameters_display(self, trans, **kwd): + def parameters_display(self, trans: ProvidesUserContext, **kwd): """ * GET /api/jobs/{job_id}/parameters_display * GET /api/datasets/{dataset_id}/parameters_display @@ -360,7 +360,7 @@ class JobController(BaseAPIController, UsesVisualizationMixin): return summarize_job_parameters(trans, job) @expose_api_anonymous - def build_for_rerun(self, trans, id, **kwd): + def build_for_rerun(self, trans: ProvidesHistoryContext, id, **kwd): """ * GET /api/jobs/{id}/build_for_rerun returns a tool input/param template prepopulated with this job's @@ -411,12 +411,12 @@ class JobController(BaseAPIController, UsesVisualizationMixin): return dataset_instance.creating_job @expose_api - def create(self, trans, payload, **kwd): + def create(self, trans: ProvidesUserContext, payload, **kwd): """ See the create method in tools.py in order to submit a job. """ raise exceptions.NotImplemented('Please POST to /api/tools instead.') @expose_api - def search(self, trans, payload, **kwd): + def search(self, trans: ProvidesHistoryContext, payload: dict, **kwd): """ search( trans, payload ) * POST /api/jobs/search: @@ -464,7 +464,7 @@ class JobController(BaseAPIController, UsesVisualizationMixin): return [self.encode_all_ids(trans, single_job.to_dict('element'), True) for single_job in jobs] @expose_api_anonymous - def error(self, trans, id, payload, **kwd): + def error(self, trans: ProvidesUserContext, id, payload, **kwd): """ error( trans, id ) * POST /api/jobs/{id}/error @@ -505,7 +505,7 @@ class JobController(BaseAPIController, UsesVisualizationMixin): @require_admin @expose_api - def show_job_lock(self, trans, **kwd): + def show_job_lock(self, trans: ProvidesUserContext, **kwd): """ * GET /api/job_lock return boolean indicating if job lock active. @@ -514,7 +514,7 @@ class JobController(BaseAPIController, UsesVisualizationMixin): @require_admin @expose_api - def update_job_lock(self, trans, payload, **kwd): + def update_job_lock(self, trans: ProvidesUserContext, payload, **kwd): """ * PUT /api/job_lock return boolean indicating if job lock active. diff --git a/lib/galaxy/webapps/galaxy/api/roles.py b/lib/galaxy/webapps/galaxy/api/roles.py index 41a42b0703d..0414bd39116 100644 --- a/lib/galaxy/webapps/galaxy/api/roles.py +++ b/lib/galaxy/webapps/galaxy/api/roles.py @@ -18,6 +18,7 @@ from pydantic import ( from galaxy import web from galaxy.app import UniverseApplication from galaxy.managers.base import decode_id +from galaxy.managers.context import ProvidesUserContext from galaxy.managers.roles import ( RoleDefinitionModel, RoleManager, @@ -25,9 +26,6 @@ from galaxy.managers.roles import ( ) from galaxy.schema.fields import EncodedDatabaseIdField from galaxy.webapps.base.controller import BaseAPIController, url_for -from galaxy.work.context import ( - SessionRequestContext, -) from . import ( get_admin_user, get_app, @@ -65,26 +63,26 @@ class FastAPIRoles: role_manager: RoleManager = Depends(get_role_manager) @router.get('/api/roles') - def index(self, trans: SessionRequestContext = Depends(get_trans)) -> RoleListModel: + def index(self, trans: ProvidesUserContext = Depends(get_trans)) -> RoleListModel: roles = self.role_manager.list_displayable_roles(trans) return RoleListModel(__root__=[role_to_model(trans, r) for r in roles]) @router.get('/api/roles/{id}') - def show(self, id: EncodedDatabaseIdField, trans: SessionRequestContext = Depends(get_trans)) -> RoleModel: + def show(self, id: EncodedDatabaseIdField, trans: ProvidesUserContext = Depends(get_trans)) -> RoleModel: role_id = trans.app.security.decode_id(id) role = self.role_manager.get(trans, role_id) return role_to_model(trans, role) @router.post("/api/roles") - def create(self, trans: SessionRequestContext = Depends(get_trans), admin_user=Depends(get_admin_user), role_definition_model: RoleDefinitionModel = Body(...)) -> RoleModel: - role = self.role_manager.create(trans, role_definition_model) + def create(self, trans: ProvidesUserContext = Depends(get_trans), admin_user=Depends(get_admin_user), role_definition_model: RoleDefinitionModel = Body(...)) -> RoleModel: + role = self.role_manager.create_role(trans, role_definition_model) return role_to_model(trans, role) class RoleAPIController(BaseAPIController): @web.expose_api - def index(self, trans, **kwd): + def index(self, trans: ProvidesUserContext, **kwd): """ GET /api/roles Displays a collection (list) of roles. @@ -93,7 +91,7 @@ class RoleAPIController(BaseAPIController): return RoleListModel(__root__=[role_to_model(trans, r) for r in roles]) @web.expose_api - def show(self, trans, id, **kwd): + def show(self, trans: ProvidesUserContext, id: str, **kwd): """ GET /api/roles/{encoded_role_id} Displays information about a role. @@ -104,14 +102,14 @@ class RoleAPIController(BaseAPIController): @web.require_admin @web.expose_api - def create(self, trans, payload, **kwd): + def create(self, trans: ProvidesUserContext, payload, **kwd): """ POST /api/roles Creates a new role. """ expand_json_keys(payload, ["user_ids", "group_ids"]) role_definition_model = RoleDefinitionModel(**payload) - role = self._role_manager.create(trans, role_definition_model) + role = self._role_manager.create_role(trans, role_definition_model) return role_to_model(trans, role) @property @@ -119,7 +117,7 @@ class RoleAPIController(BaseAPIController): return self.app.role_manager -def expand_json_keys(payload, keys): +def expand_json_keys(payload: dict, keys): for key in keys: value = payload.get(key) if isinstance(value, str): diff --git a/lib/galaxy/work/context.py b/lib/galaxy/work/context.py index 776b74b0601..0aeb3041ab8 100644 --- a/lib/galaxy/work/context.py +++ b/lib/galaxy/work/context.py @@ -1,11 +1,9 @@ from galaxy.managers.context import ( - ProvidesAppContext, ProvidesHistoryContext, - ProvidesUserContext ) -class WorkRequestContext(ProvidesAppContext, ProvidesUserContext, ProvidesHistoryContext): +class WorkRequestContext(ProvidesHistoryContext): """ Stripped down implementation of Galaxy web transaction god object for work request handling outside of web threads - uses mix-ins shared with GalaxyWebTransaction to provide app, user, and history context convenience @@ -19,13 +17,16 @@ class WorkRequestContext(ProvidesAppContext, ProvidesUserContext, ProvidesHistor """ def __init__(self, app, user=None, history=None, workflow_building_mode=False): - self.app = app - self.security = app.security + self._app = app self.__user = user self.__user_current_roles = None self.__history = history self.workflow_building_mode = workflow_building_mode + @property + def app(self): + return self._app + def get_history(self, create=False): return self.__history diff --git a/test/integration/test_workflow_refactoring.py b/test/integration/test_workflow_refactoring.py index 852484cf8d4..d842e738b94 100644 --- a/test/integration/test_workflow_refactoring.py +++ b/test/integration/test_workflow_refactoring.py @@ -826,10 +826,10 @@ def _step_with_label(native_dict, label): class MockTrans(ProvidesAppContext): def __init__(self, app, user): - self.app = app + self._app = app self.user = user self.history = None @property - def security(self): - return self.app.security + def app(self): + return self._app diff --git a/test/unit/test_galaxy_transactions.py b/test/unit/test_galaxy_transactions.py index 3ab458980ee..d3214ff8b52 100644 --- a/test/unit/test_galaxy_transactions.py +++ b/test/unit/test_galaxy_transactions.py @@ -7,7 +7,11 @@ from galaxy.util import bunch class TestTransaction(context.ProvidesAppContext): def __init__(self): - self.app = TestApp() + self._app = TestApp() + + @property + def app(self): + return self._app def test_logging_events_off(): From 2a2559812c0a240cf03cfaeb56c784cdcab0315e Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 12 Jan 2021 20:45:19 -0500 Subject: [PATCH 096/208] Define __all__ list for tours package --- lib/galaxy/tours/__init__.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/tours/__init__.py b/lib/galaxy/tours/__init__.py index 060f7f161a9..b8549dc9a45 100644 --- a/lib/galaxy/tours/__init__.py +++ b/lib/galaxy/tours/__init__.py @@ -1,7 +1,17 @@ -from ._impl import build_tours_registry # noqa -from ._interface import ToursRegistry # noqa -from ._schema import Tour # noqa -from ._schema import TourCore # noqa -from ._schema import TourStep # noqa -from ._schema import TourDetails # noqa -from ._schema import TourList # noqa +from ._impl import build_tours_registry +from ._interface import ToursRegistry +from ._schema import Tour +from ._schema import TourCore +from ._schema import TourDetails +from ._schema import TourList +from ._schema import TourStep + +__all__ = [ + 'build_tours_registry', + 'ToursRegistry', + 'Tour', + 'TourCore', + 'TourDetails', + 'TourList', + 'TourStep' +] From 02e9f02db9af0118658f1183a2770283e1798f24 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 13 Jan 2021 14:04:14 +0100 Subject: [PATCH 097/208] Refactor move pydantic model to datatypes/_schema.py --- lib/galaxy/datatypes/_schema.py | 129 +++++++++++++++++++++ lib/galaxy/managers/datatypes.py | 129 +-------------------- lib/galaxy/webapps/galaxy/api/datatypes.py | 4 +- 3 files changed, 138 insertions(+), 124 deletions(-) create mode 100644 lib/galaxy/datatypes/_schema.py diff --git a/lib/galaxy/datatypes/_schema.py b/lib/galaxy/datatypes/_schema.py new file mode 100644 index 00000000000..6efb7cc4565 --- /dev/null +++ b/lib/galaxy/datatypes/_schema.py @@ -0,0 +1,129 @@ +from typing import ( + Dict, + List, + Optional, +) + +from pydantic import ( + BaseModel, + Field, + HttpUrl +) + + +class CompositeFileInfo(BaseModel): + name: str = Field( + ..., # Mark this field as required + title="Name", + description="The name of this composite file" + ) + optional: bool = Field( + title="Optional", + description="" # TODO add description + ) + mimetype: Optional[str] = Field( + title="MIME type", + description="The MIME type of this file" + ) + description: Optional[str] = Field( + title="Description", + description="Summary description of the purpouse of this file" + ) + substitute_name_with_metadata: Optional[str] = Field( + title="Substitute name with metadata", + description="" # TODO add description + ) + is_binary: bool = Field( + title="Is binary", + description="Whether this file is a binary file" + ) + to_posix_lines: bool = Field( + title="To posix lines", + description="" # TODO add description + ) + space_to_tab: bool = Field( + title="Spaces to tabulation", + description="" # TODO add description + ) + + +class DatatypeDetails(BaseModel): + extension: str = Field( + ..., # Mark this field as required + title="Extension", + description="The data type’s Dataset file extension", + example="bed" + ) + description: Optional[str] = Field( + title="Description", + description="A summary description for this data type" + ) + description_url: Optional[HttpUrl] = Field( + title="Description URL", + description="The URL to a detailed description for this datatype", + example="https://wiki.galaxyproject.org/Learn/Datatypes#Bed" + ) + display_in_upload: bool = Field( + default=False, + title="Display in upload", + description="If True, the associated file extension will be displayed in the `File Format` select list in the `Upload File from your computer` tool in the `Get Data` tool section of the tool panel" + ) + composite_files: Optional[List[CompositeFileInfo]] = Field( + default=None, + title="Composite files", + description="A collection of files composing this data type" + ) + + +class DatatypesMap(BaseModel): + ext_to_class_name: Dict[str, str] = Field( + ..., # Mark this field as required + title="Extension Map", + description="Dictionary mapping datatype's extensions with implementation classes", + ) + class_to_classes: Dict[str, Dict[str, bool]] = Field( + ..., # Mark this field as required + title="Classes Map", + description="Dictionary mapping datatype's classes with their base classes", + ) + + +class DatatypesCombinedMap(BaseModel): + datatypes: List[str] = Field( + ..., # Mark this field as required + title="Datatypes", + description="List of datatypes extensions", + ) + datatypes_mapping: DatatypesMap = Field( + ..., # Mark this field as required + title="Datatypes Mapping", + description="Dictionaries for mapping datatype's extensions/classes with their implementation classes", + ) + + +class DatatypeConverter(BaseModel): + source: str = Field( + ..., # Mark this field as required + title="Source", + description="Source type for conversion", + example="bam", + ) + target: str = Field( + ..., # Mark this field as required + title="Target", + description="Target type for conversion", + example="bai", + ) + tool_id: str = Field( + ..., # Mark this field as required + title="Tool identifier", + description="The converter tool identifier", + example="CONVERTER_Bam_Bai_0", + ) + + +class DatatypeConverterList(BaseModel): + __root__: List[DatatypeConverter] = Field( + title='List of data type converters', + default=[] + ) diff --git a/lib/galaxy/managers/datatypes.py b/lib/galaxy/managers/datatypes.py index edce2737e17..6913bc32104 100644 --- a/lib/galaxy/managers/datatypes.py +++ b/lib/galaxy/managers/datatypes.py @@ -5,135 +5,18 @@ from typing import ( Union, ) -from pydantic import ( - BaseModel, - Field, - HttpUrl -) from pydantic.tools import parse_obj_as +from galaxy.datatypes._schema import ( + DatatypeConverterList, + DatatypeDetails, + DatatypesCombinedMap, + DatatypesMap, +) from galaxy.datatypes.data import Data from galaxy.datatypes.registry import Registry -class CompositeFileInfo(BaseModel): - name: str = Field( - ..., # Mark this field as required - title="Name", - description="The name of this composite file" - ) - optional: bool = Field( - title="Optional", - description="" # TODO add description - ) - mimetype: Optional[str] = Field( - title="MIME type", - description="The MIME type of this file" - ) - description: Optional[str] = Field( - title="Description", - description="Summary description of the purpouse of this file" - ) - substitute_name_with_metadata: Optional[str] = Field( - title="Substitute name with metadata", - description="" # TODO add description - ) - is_binary: bool = Field( - title="Is binary", - description="Whether this file is a binary file" - ) - to_posix_lines: bool = Field( - title="To posix lines", - description="" # TODO add description - ) - space_to_tab: bool = Field( - title="Spaces to tabulation", - description="" # TODO add description - ) - - -class DatatypeDetails(BaseModel): - extension: str = Field( - ..., # Mark this field as required - title="Extension", - description="The data type’s Dataset file extension", - example="bed" - ) - description: Optional[str] = Field( - title="Description", - description="A summary description for this data type" - ) - description_url: Optional[HttpUrl] = Field( - title="Description URL", - description="The URL to a detailed description for this datatype", - example="https://wiki.galaxyproject.org/Learn/Datatypes#Bed" - ) - display_in_upload: bool = Field( - default=False, - title="Display in upload", - description="If True, the associated file extension will be displayed in the `File Format` select list in the `Upload File from your computer` tool in the `Get Data` tool section of the tool panel" - ) - composite_files: Optional[List[CompositeFileInfo]] = Field( - default=None, - title="Composite files", - description="A collection of files composing this data type" - ) - - -class DatatypesMap(BaseModel): - ext_to_class_name: Dict[str, str] = Field( - ..., # Mark this field as required - title="Extension Map", - description="Dictionary mapping datatype's extensions with implementation classes", - ) - class_to_classes: Dict[str, Dict[str, bool]] = Field( - ..., # Mark this field as required - title="Classes Map", - description="Dictionary mapping datatype's classes with their base classes", - ) - - -class DatatypesCombinedMap(BaseModel): - datatypes: List[str] = Field( - ..., # Mark this field as required - title="Datatypes", - description="List of datatypes extensions", - ) - datatypes_mapping: DatatypesMap = Field( - ..., # Mark this field as required - title="Datatypes Mapping", - description="Dictionaries for mapping datatype's extensions/classes with their implementation classes", - ) - - -class DatatypeConverter(BaseModel): - source: str = Field( - ..., # Mark this field as required - title="Source", - description="Source type for conversion", - example="bam", - ) - target: str = Field( - ..., # Mark this field as required - title="Target", - description="Target type for conversion", - example="bai", - ) - tool_id: str = Field( - ..., # Mark this field as required - title="Tool identifier", - description="The converter tool identifier", - example="CONVERTER_Bam_Bai_0", - ) - - -class DatatypeConverterList(BaseModel): - __root__: List[DatatypeConverter] = Field( - title='List of data type converters', - default=[] - ) - - def view_index( datatypes_registry: Registry, extension_only: Optional[bool] = True, diff --git a/lib/galaxy/webapps/galaxy/api/datatypes.py b/lib/galaxy/webapps/galaxy/api/datatypes.py index 178cf870b2c..55d748b886b 100644 --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -23,7 +23,9 @@ from galaxy.managers.datatypes import ( DatatypeDetails, DatatypesCombinedMap, DatatypesMap, - view_converters, view_edam_data, view_edam_formats, + view_converters, + view_edam_data, + view_edam_formats, view_index, view_mapping, view_sniffers, From e91e754ac8e2bf2d1858eeec01b77c7581fb304a Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 13 Jan 2021 14:31:16 +0100 Subject: [PATCH 098/208] Add pydantic to the packages/data/requirements.txt --- packages/data/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/data/requirements.txt b/packages/data/requirements.txt index 98424dcbef5..85c7facff5b 100644 --- a/packages/data/requirements.txt +++ b/packages/data/requirements.txt @@ -8,6 +8,7 @@ isa-rwval parsley numpy<=1.16 pycryptodome +pydantic pysam social_auth_core SQLAlchemy From fa685e4fd089226e5039e44e49995290c014c4a4 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 12 Jan 2021 22:30:11 -0500 Subject: [PATCH 099/208] uvicorn-compatible factory function for fastapi. --- lib/galaxy/main_config.py | 36 ++++++++++ lib/galaxy/webapps/galaxy/fast_factory.py | 82 +++++++++++++++++++++++ packages/app/galaxy/main_config.py | 1 + scripts/galaxy_main.py | 36 +++------- 4 files changed, 127 insertions(+), 28 deletions(-) create mode 100644 lib/galaxy/main_config.py create mode 100644 lib/galaxy/webapps/galaxy/fast_factory.py create mode 120000 packages/app/galaxy/main_config.py diff --git a/lib/galaxy/main_config.py b/lib/galaxy/main_config.py new file mode 100644 index 00000000000..c1f981ab148 --- /dev/null +++ b/lib/galaxy/main_config.py @@ -0,0 +1,36 @@ +"""Utilities for finding Galaxy's configuration file. + +This is for use by web framework code and scripts (e.g. scripts/galaxy_main.py). +""" +import os + +DEFAULT_INIS = ["config/galaxy.yml", "config/galaxy.ini", "universe_wsgi.ini", "config/galaxy.yml.sample"] +DEFAULT_INI_APP = "main" +DEFAULT_CONFIG_SECTION = "galaxy" + + +def absolute_config_path(path, galaxy_root): + if path and not os.path.isabs(path): + path = os.path.join(galaxy_root, path) + return path + + +def config_is_ini(config_file): + return config_file and (config_file.endswith('.ini') or config_file.endswith('.ini.sample')) + + +def find_config(supplied_config, galaxy_root): + if supplied_config: + return supplied_config + + if galaxy_root is None: + return os.path.abspath('galaxy.yml') + + # If not explicitly supplied an config, check galaxy.ini and then + # just resort to sample if that has not been configured. + for guess in DEFAULT_INIS: + config_path = os.path.join(galaxy_root, guess) + if os.path.exists(config_path): + return config_path + + return guess diff --git a/lib/galaxy/webapps/galaxy/fast_factory.py b/lib/galaxy/webapps/galaxy/fast_factory.py new file mode 100644 index 00000000000..4480c7a397f --- /dev/null +++ b/lib/galaxy/webapps/galaxy/fast_factory.py @@ -0,0 +1,82 @@ +"""Module containing factory class for building uvicorn app for Galaxy. + +Information on uvicorn, its various settings, and how to invoke it can +be found at https://www.uvicorn.org/. + +Galaxy can be launched with uvicorn using the following invocation: + +:: + + uvicorn --app-dir lib --factory galaxy.webapps.galaxy.fast_factory:factory + +Use the environment variable ``GALAXY_CONFIG_FILE`` to specify a Galaxy +configuration file. Galaxy configuration can be loading from a YAML +or an .ini file (reads app:main currently but can be overridden with +GALAXY_CONFIG_SECTION). + +:: + + GALAXY_CONFIG_FILE=config/galaxy.yml uvicorn --app-dir lib --factory galaxy.webapps.galaxy.fast_factory:factory + +.. note:: + + Information on additional ways to configure uvicorn can be found at + https://www.uvicorn.org/. + +.. warning:: + + If an ini file is supplied via GALAXY_CONFIG_FILE, the server properties + such as host and port are not read from the file like older forms of + configuring Galaxy. + +`Gunicorn `__ is a server with +more complex management options. + +This factory function can be executed as a uvicorn worker managed with gunicorn +with the following command-line. + +:: + + gunicorn 'galaxy.webapps.galaxy.fast_factory:factory()' --env GALAXY_CONFIG_FILE=config/galaxy.ini --pythonpath lib -w 4 -k uvicorn.workers.UvicornWorker + +""" +import os + +from galaxy.main_config import ( + absolute_config_path, + config_is_ini, + DEFAULT_CONFIG_SECTION, + DEFAULT_INI_APP, + find_config, +) +from galaxy.web_stack import get_app_kwds +from galaxy.webapps.galaxy.buildapp import app_factory +from .fast_app import initialize_fast_app + + +def factory(): + kwds = get_app_kwds("galaxy", "galaxy") + config_file = kwds.get("config_file") + if not config_file and "GALAXY_CONFIG_FILE" in os.environ: + config_file = os.path.abspath(os.environ["GALAXY_CONFIG_FILE"]) + else: + galaxy_root = kwds.get("galaxy_root") + config_file = find_config(config_file, galaxy_root) + config_file = absolute_config_path(config_file, galaxy_root=galaxy_root) + + if "GALAXY_CONFIG_SECTION" in os.environ: + config_section = os.environ["GALAXY_CONFIG_SECTION"] + elif config_is_ini(config_file): + config_section = "app:%s" % DEFAULT_INI_APP + else: + config_section = DEFAULT_CONFIG_SECTION + + if 'config_file' not in kwds: + kwds['config_file'] = config_file + if 'config_section' not in kwds: + kwds['config_section'] = config_section + global_conf = {} + if config_is_ini(config_file): + global_conf["__file__"] = config_file + gx = app_factory(global_conf=global_conf, load_app_kwds=kwds) + return initialize_fast_app(gx) diff --git a/packages/app/galaxy/main_config.py b/packages/app/galaxy/main_config.py new file mode 120000 index 00000000000..f26b2577893 --- /dev/null +++ b/packages/app/galaxy/main_config.py @@ -0,0 +1 @@ +../../../lib/galaxy/main_config.py \ No newline at end of file diff --git a/scripts/galaxy_main.py b/scripts/galaxy_main.py index cc4cdea7374..c6a49f20bcc 100755 --- a/scripts/galaxy_main.py +++ b/scripts/galaxy_main.py @@ -57,15 +57,18 @@ else: except Exception: log.exception("Failed to add Galaxy to sys.path") raise +from galaxy.main_config import ( + absolute_config_path, + config_is_ini, + DEFAULT_CONFIG_SECTION, + DEFAULT_INI_APP, + find_config, +) from galaxy.util import unicodify from galaxy.web_stack import get_app_kwds REQUIRES_DAEMONIZE_MESSAGE = "Attempted to use Galaxy in daemon mode, but daemonize is unavailable." -DEFAULT_INI_APP = "main" -DEFAULT_CONFIG_SECTION = "galaxy" -DEFAULT_INIS = ["config/galaxy.yml", "config/galaxy.ini", "universe_wsgi.ini", "config/galaxy.yml.sample"] - DEFAULT_PID = "galaxy.pid" DEFAULT_VERBOSE = True DESCRIPTION = "Daemonized entry point for Galaxy." @@ -156,29 +159,6 @@ def app_loop(args, log): raise -def absolute_config_path(path, galaxy_root): - if path and not os.path.isabs(path): - path = os.path.join(galaxy_root, path) - return path - - -def find_config(supplied_config, galaxy_root): - if supplied_config: - return supplied_config - - if galaxy_root is None: - return os.path.abspath('galaxy.yml') - - # If not explicitly supplied an config, check galaxy.ini and then - # just resort to sample if that has not been configured. - for guess in DEFAULT_INIS: - config_path = os.path.join(galaxy_root, guess) - if os.path.exists(config_path): - return config_path - - return guess - - class GalaxyConfigBuilder: """ Generate paste-like configuration from supplied command-line arguments. """ @@ -218,7 +198,7 @@ class GalaxyConfigBuilder: @property def config_is_ini(self): - return self.config_file.endswith('.ini') or self.config_file.endswith('.ini.sample') + return config_is_ini(self.config_file) def app_kwds(self): kwds = get_app_kwds(self.app_name, app_name=self.app_name) From 0492602e802e474566dd32d44136eed5b4d6890d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 13 Jan 2021 21:11:32 +0100 Subject: [PATCH 100/208] Update version to 21.05.dev --- lib/galaxy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/version.py b/lib/galaxy/version.py index 93b5057948d..803342840aa 100644 --- a/lib/galaxy/version.py +++ b/lib/galaxy/version.py @@ -1,3 +1,3 @@ -VERSION_MAJOR = "21.01" +VERSION_MAJOR = "21.05" VERSION_MINOR = "dev" VERSION = VERSION_MAJOR + ('.' + VERSION_MINOR if VERSION_MINOR else '') From e5e7b6258df85d4e38b17dd88e64bdbbeb8401ef Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 13 Jan 2021 21:11:32 +0100 Subject: [PATCH 101/208] Update version to 21.01.rc1 --- lib/galaxy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/version.py b/lib/galaxy/version.py index 93b5057948d..bd95068cd1a 100644 --- a/lib/galaxy/version.py +++ b/lib/galaxy/version.py @@ -1,3 +1,3 @@ VERSION_MAJOR = "21.01" -VERSION_MINOR = "dev" +VERSION_MINOR = "rc1" VERSION = VERSION_MAJOR + ('.' + VERSION_MINOR if VERSION_MINOR else '') From 21f06bafef2c4f5403cfe5bf1067e5ec774944ac Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 12 Jan 2021 20:15:20 -0500 Subject: [PATCH 102/208] API design guidelines. --- doc/source/api/guidelines.rst | 70 ----------- doc/source/api/quickstart.rst | 186 +----------------------------- doc/source/api_doc.rst | 3 - doc/source/dev/api_guidelines.rst | 172 +++++++++++++++++++++++++++ doc/source/dev/index.rst | 1 + 5 files changed, 176 insertions(+), 256 deletions(-) delete mode 100644 doc/source/api/guidelines.rst create mode 100644 doc/source/dev/api_guidelines.rst diff --git a/doc/source/api/guidelines.rst b/doc/source/api/guidelines.rst deleted file mode 100644 index 4260144da2f..00000000000 --- a/doc/source/api/guidelines.rst +++ /dev/null @@ -1,70 +0,0 @@ -API Design Guidelines -===================== - -The following section outlines guidelines related to extending and/or modifying -the Galaxy API. The Galaxy API has grown in an ad-hoc fashion over time by -many contributors and so clients SHOULD NOT expect the API will conform to -these guidelines - but developers contributing to the Galaxy API SHOULD follow -these guidelines. - - - API functionality should include docstring documentation for consumption - at docs.galaxyproject.org. - - Developers should familiarize themselves with the HTTP status code definitions - http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html. The API responses - should properly set the status code according to the result - in particular - 2XX responses should be used for successful requests, 4XX for various - kinds of client errors, and 5XX for the errors on the server side. - - If there is an error processing some part of the request (one item in a list - for instance), the status code should be set to reflect the error and the - partial result may or may not be returned depending on the controller - - this behavior should be documented. - - API methods should throw a finite number of exceptions - (defined in :mod:`galaxy.exceptions`) and these should subclass - `MessageException` and not paste/wsgi HTTP exceptions. When possible, - the framework itself should be responsible for catching these exceptions, - setting the status code, and building an error response. - - Error responses should not consist of plain text strings - they should be - dictionaries describing the error and containing the following:: - - { - "status_code": 400, - "err_code": 400007, - "err_msg": "Request contained invalid parameter, action could not be completed.", - "type": "error", - "extra_error_info": "Extra information." - } - - Various error conditions (once a format has been chosen and framework to - enforce it in place) should be spelled out in this document. - - Backward compatibility is important and should be maintained when possible. - If changing behavior in a non-backward compatible way please ensure one - of the following holds - there is a strong reason to believe no consumers - depend on a behavior, the behavior is effectively broken, or the API - method being modified has not been part of a tagged dist release. - -The following bullet points represent good practices more than guidelines, please -consider them when modifying the API. - - - Functionality should not be copied and pasted between controllers - - consider refactoring functionality into associated classes or short of - that into Mixins (http://en.wikipedia.org/wiki/Composition_over_inheritance) - or into Managers (:mod:`galaxy.managers`). - - API additions are more permanent changes to Galaxy than many other potential - changes and so a second opinion on API changes should be sought. - - New API functionality should include functional tests. These functional - tests should be implemented in Python and placed in - `lib/galaxy_test/api`. - - Changes to reflect modifications to the API should be pushed upstream to - the BioBlend project if possible. - -Longer term goals/notes. - - - It would be advantageous to have a clearer separation of anonymous and - admin handling functionality. - - If at some point in the future, functionality needs to be added that - breaks backward compatibility in a significant way to a component used by - the community - a "dev" variant of the API will be established and - the community should be alerted and given a timeframe for when the old - behavior will be replaced with the new behavior. - - Consistent standards for range-based requests, batch requests, filtered - requests, etc... should be established and documented here. diff --git a/doc/source/api/quickstart.rst b/doc/source/api/quickstart.rst index bd517ba461d..d1a251cf7ae 100644 --- a/doc/source/api/quickstart.rst +++ b/doc/source/api/quickstart.rst @@ -1,186 +1,6 @@ Quickstart ========== -Log in as your user, navigate to the API Keys page in the User menu, and -generate a new API key. Make a note of the API key, and then pull up a -terminal. Now we'll use the display.py script in your ``galaxy/scripts/api`` -directory for a short example, which assumes your local galaxy server is -running on the default port 8080:: - - % ./display.py my_key http://localhost:8080/api/histories - Collection Members - ------------------ - #1: /api/histories/8c49be448cfe29bc - name: Unnamed history - id: 8c49be448cfe29bc - #2: /api/histories/33b43b4e7093c91f - name: output test - id: 33b43b4e7093c91f - -The result is a Collection of the histories of the user specified by the API -key (you). To look at the details of a particular history, say #1 above, do -the following:: - - % ./display.py my_key http://localhost:8080/api/histories/8c49be448cfe29bc - Member Information - ------------------ - state_details: {'ok': 1, 'failed_metadata': 0, 'upload': 0, 'discarded': 0, 'running': 0, 'setting_metadata': 0, 'error': 0, 'new': 0, 'queued': 0, 'empty': 0} - state: ok - contents_url: /api/histories/8c49be448cfe29bc/contents - id: 8c49be448cfe29bc - name: Unnamed history - -This gives detailed information about the specific member in question, in this -case the History. To view history contents, do the following:: - - - % ./display.py my_key http://localhost:8080/api/histories/8c49be448cfe29bc/contents - Collection Members - ------------------ - #1: /api/histories/8c49be448cfe29bc/contents/6f91353f3eb0fa4a - name: Pasted Entry - type: file - id: 6f91353f3eb0fa4a - -What we have here is another Collection of items containing all of the datasets -in this particular history. Finally, to view details of a particular dataset -in this collection, execute the following:: - - % ./display.py my_key http://localhost:8080/api/histories/8c49be448cfe29bc/contents/6f91353f3eb0fa4a - Member Information - ------------------ - misc_blurb: 1 line - name: Pasted Entry - data_type: txt - deleted: False - file_name: /Users/yoplait/work/galaxy-stock/database/files/000/dataset_82.dat - state: ok - download_url: /datasets/6f91353f3eb0fa4a/display?to_ext=txt - visible: True - genome_build: ? - model_class: HistoryDatasetAssociation - file_size: 17 - metadata_data_lines: 1 - id: 6f91353f3eb0fa4a - misc_info: uploaded txt file - metadata_dbkey: ? - -And now you've successfully used the API to request and select a history, -browse the contents of that history, and then look at detailed information -about a particular dataset. - -For a more comprehensive Data Library example, set the following options in your -galaxy.yml as well, and restart galaxy again:: - - admin_users: you@example.org - library_import_dir: /path/to/some/directory - -In the directory you specified for 'library_import_dir', create some -subdirectories, and put (or symlink) files to import into Galaxy into those -subdirectories. - -In Galaxy, create an account that matches the address you put in 'admin_users', -then browse to that user's preferences and generate a new API Key. Copy the -key to your clipboard and then use these scripts:: - - % ./display.py my_key http://localhost:8080/api/libraries - Collection Members - ------------------ - - 0 elements in collection - - % ./library_create_library.py my_key http://localhost:8080/api/libraries api_test 'API Test Library' - Response - -------- - /api/libraries/f3f73e481f432006 - name: api_test - id: f3f73e481f432006 - - % ./display.py my_key http://localhost:8080/api/libraries - Collection Members - ------------------ - /api/libraries/f3f73e481f432006 - name: api_test - id: f3f73e481f432006 - - % ./display.py my_key http://localhost:8080/api/libraries/f3f73e481f432006 - Member Information - ------------------ - synopsis: None - contents_url: /api/libraries/f3f73e481f432006/contents - description: API Test Library - name: api_test - - % ./display.py my_key http://localhost:8080/api/libraries/f3f73e481f432006/contents - Collection Members - ------------------ - /api/libraries/f3f73e481f432006/contents/28202595c0d2591f61ddda595d2c3670 - name: / - type: folder - id: 28202595c0d2591f61ddda595d2c3670 - - % ./library_create_folder.py my_key http://localhost:8080/api/libraries/f3f73e481f432006/contents 28202595c0d2591f61ddda595d2c3670 api_test_folder1 'API Test Folder 1' - Response - -------- - /api/libraries/f3f73e481f432006/contents/28202595c0d2591fa4f9089d2303fd89 - name: api_test_folder1 - id: 28202595c0d2591fa4f9089d2303fd89 - - % ./library_upload_from_import_dir.py my_key http://localhost:8080/api/libraries/f3f73e481f432006/contents 28202595c0d2591fa4f9089d2303fd89 bed bed hg19 - Response - -------- - /api/libraries/f3f73e481f432006/contents/e9ef7fdb2db87d7b - name: 2.bed - id: e9ef7fdb2db87d7b - /api/libraries/f3f73e481f432006/contents/3b7f6a31f80a5018 - name: 3.bed - id: 3b7f6a31f80a5018 - - % ./display.py my_key http://localhost:8080/api/libraries/f3f73e481f432006/contents - Collection Members - ------------------ - /api/libraries/f3f73e481f432006/contents/28202595c0d2591f61ddda595d2c3670 - name: / - type: folder - id: 28202595c0d2591f61ddda595d2c3670 - /api/libraries/f3f73e481f432006/contents/28202595c0d2591fa4f9089d2303fd89 - name: /api_test_folder1 - type: folder - id: 28202595c0d2591fa4f9089d2303fd89 - /api/libraries/f3f73e481f432006/contents/e9ef7fdb2db87d7b - name: /api_test_folder1/2.bed - type: file - id: e9ef7fdb2db87d7b - /api/libraries/f3f73e481f432006/contents/3b7f6a31f80a5018 - name: /api_test_folder1/3.bed - type: file - id: 3b7f6a31f80a5018 - - % ./display.py my_key http://localhost:8080/api/libraries/f3f73e481f432006/contents/e9ef7fdb2db87d7b - Member Information - ------------------ - misc_blurb: 68 regions - metadata_endCol: 3 - data_type: bed - metadata_columns: 6 - metadata_nameCol: 4 - uploaded_by: nate@... - metadata_strandCol: 6 - name: 2.bed - genome_build: hg19 - metadata_comment_lines: None - metadata_startCol: 2 - metadata_chromCol: 1 - file_size: 4272 - metadata_data_lines: 68 - message: - metadata_dbkey: hg19 - misc_info: uploaded bed file - date_uploaded: 2010-06-22T17:01:51.266119 - metadata_column_types: str, int, int, str, int, str - -Other parameters are valid when uploading, they are the same parameters as are -used in the web form, like 'link_data_only' and etc. - -The request and response format should be considered alpha and are subject to change. - +The Galaxy Training Network material has a tutorial designed +to teach using the Galaxy API at +https://training.galaxyproject.org/training-material/topics/dev/tutorials/bioblend-api/slides.html. diff --git a/doc/source/api_doc.rst b/doc/source/api_doc.rst index 379cb8d006b..85e56bc2afe 100644 --- a/doc/source/api_doc.rst +++ b/doc/source/api_doc.rst @@ -28,7 +28,4 @@ Overview of available API resources is available at https://galaxyproject.org/de Quickstart - Development Guidelines - Documentation - diff --git a/doc/source/dev/api_guidelines.rst b/doc/source/dev/api_guidelines.rst new file mode 100644 index 00000000000..fe3b027f4b6 --- /dev/null +++ b/doc/source/dev/api_guidelines.rst @@ -0,0 +1,172 @@ +API Design Guidelines +===================== + +The following section outlines guidelines related to extending and/or modifying +the Galaxy API. The Galaxy API has grown in an ad-hoc fashion over time by +many contributors and so clients SHOULD NOT expect the API will conform to +these guidelines - but developers contributing to the Galaxy API SHOULD follow +these guidelines. + +Before Getting Started +~~~~~~~~~~~~~~~~~~~~~~ + +Developers should familiarize themselves with HTTP method definitions +https://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html#sec9. While Galaxy +follows these imperfectly - a serious effort should be made to use HTTP +methods (GET, POST, PUT, DELETE, etc.) correctly and consistently. More +information about these methods with a focus on RESTful APIs can be found +at https://restfulapi.net/http-methods/. + +Developers should familiarize themselves with the HTTP status code definitions +http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html. The API responses +should properly set the status code according to the result - in particular +2XX responses should be used for successful requests, 4XX for various +kinds of client errors, and 5XX for the errors on the server side. + +Developers should also familiarize themselves with RESTful API design +https://docs.microsoft.com/en-us/azure/architecture/best-practices/api-design. +Again Galaxy follows generic RESTful guidelines imperfectly - but a serious +effort should be made to design the API as RESTfully and consistently as possible. + +Documenting the API +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +An API endpoint implementation method should include high-level docstring documentation +for consumption at docs.galaxyproject.org. + +API functionality should consume and produce Pydantic_ `models +`__ with typed and annotated +`fields `__. This information +will be used to generate detailed OpenAPI_ documentation for APIs in +the near future. Typed models also improve the quality and thoroughness +of the static checking that can be done for the Galaxy backend. + +To further document your API and allow other developers to see how it functions, +new API functionality should include functional tests. These functional +tests should be implemented in Python and added to either the API test suite +or the integration test suite. Checkout the :ref:`Writing Tests for Galaxy <./writing_tests.md>` +documentation for more information on developing tests for Galaxy. + +.. note:: + + The Galaxy documentation can be generated by running ``make docs-develop`` + from the Galaxy root. Galaxy can be type checked by running ``tox -e mypy`` + from the root of Galaxy after tox_ has been configured. + +Error Handling +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +API methods should throw a finite number of exceptions +(defined in :mod:`galaxy.exceptions`) and these should subclass +:class:`galaxy.exceptions.MessageException` and not paste/WSGI/FastAPI HTTP exceptions. +When possible, the framework itself should be responsible for catching these exceptions, +setting the status code, and building an error response. Throwing these exceptions +will ensure the API responses are consistent across frameworks and across the API. +These exceptions allow throwers to attach arbitrary extra information. + +Error responses should not consist of plain text strings - they should be +dictionaries describing the error and containing the following: + +.. code-block: json + + { + "status_code": 400, + "err_code": 400007, + "err_msg": "Request contained invalid parameter, action could not be completed.", + "type": "error", + "extra_error_info": "Extra information." + } + +The error code (``err_code``) is defined by the exception class +and pulled from ``lig/galaxy/exceptions/error_codes.json``. This JSON +document should be used to add new error conditions or improve +default error messages. + +If there is an error processing some part of the request (one item in a list +for instance), the status code should be set to reflect the error and the +partial result may or may not be returned depending on the controller - +this behavior should be documented. + +Backward Compatibility +~~~~~~~~~~~~~~~~~~~~~~ + +Backward compatibility is important and should be maintained when possible. +If changing behavior in a non-backward compatible way please ensure one +of the following holds - there is a strong reason to believe no consumers +depend on a behavior, the behavior is effectively broken, or the API +method being modified has not been part of a tagged dist release. If these +conditions do not hold, please help the Galaxy release manager ensure +information about the breaking API is highlighted in the Galaxy release +notes. + +Whenever functionality needs to be added, removed, or modified that +breaks backward compatibility in a significant way to a component used by +the community - a "dev" variant of the API should be established and +the community should be alerted and given a time frame for when the old +behavior will be replaced with the new behavior. + +Design +~~~~~~ + +Functionality should not be copied and pasted between controllers - +consider refactoring functionality into associated classes or short of +that into mixins_ or into managers (:mod:`galaxy.managers`). + +To whatever degree is possible, the Galaxy API controller classes +(found in :mod:`galaxy.webapps.galaxy.api`) should be as lean as +possible. Most of the logic should be pushed into components that +are decoupled from the actual web application and the web framework. +The role of the controller should be simply to adapt the web request +(tied to the web framework) to simple objects (plain Python objects, +Pydantic_ models, or SQL Alchemy models), send these objects to a +Python component lower in the stack, and finally to translate the +response to a web response. This design will ensure that Galaxy +application logic is usable in as many contexts as possible and +decouple the core "business logic" of Galaxy from the web framework +hosting it. + +Going forward, we anticipate the most typical way this abstract +advice translates to concrete code flow is that API controllers +in :mod:`galaxy.webapps.galaxy.api` receive or produce Pydantic_ +models and pass them to manager components defined in +:mod:`galaxy.managers`. The manager is then responsible for +interacting with the whole backend or specific components to +fullfil the desired request. + +Galaxy work contexts provide a way for API requests (and other +components) to describe what they require from the framework +in terms of the request (examples include the Galaxy app, a user +to act on behalf of, or a user with a defined "current history"). +These objects describing the web request are passed around rather +haphazardly as an argument named ``trans``. To design API methods +to deal with ``trans`` in the most statically checkable and +rigorous way - please review the docs for :mod:`galaxy.managers.context`. + +Examples +~~~~~~~~ + +A very clean, isolated example of a well documented controller +and well annotated models is the licenses endpoint - +:mod:`galaxy.webapps.galaxy.api.licenses`. A more complicated +endpoint that demonstrates working with a manager and the +Galaxy model backend is the roles endpoint - :mod:`galaxy.webapps.galaxy.api.roles`. + +BioBlend +~~~~~~~~ + +Changes to reflect modifications to the API should be pushed upstream to +the BioBlend_ project if possible. + +Longer Term Goals +~~~~~~~~~~~~~~~~~~ + + - It would be advantageous to have a clearer separation of anonymous and + admin handling functionality. + - Consistent standards for range-based requests, batch requests, filtered + requests, etc... should be established and documented here. + +.. _Pydantic: https://pydantic-docs.helpmanual.io/ +.. _BioBlend: https://github.com/galaxyproject/bioblend +.. _mixins: http://en.wikipedia.org/wiki/Composition_over_inheritance +.. _tox: https://tox.readthedocs.io/en/latest/ +.. _OpenAPI: https://www.openapis.org diff --git a/doc/source/dev/index.rst b/doc/source/dev/index.rst index d1d16374be1..74f48f5be70 100644 --- a/doc/source/dev/index.rst +++ b/doc/source/dev/index.rst @@ -12,6 +12,7 @@ A multi-hour long video playlist covering these slides can be found at :maxdepth: 1 schema + api_guidelines interactive_environments build_a_job_runner finding_and_improving_slow_code From 400da5d67716393879166f1950ed3f5e52dc33fb Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 13 Jan 2021 13:25:49 -0500 Subject: [PATCH 103/208] Apply suggestions from code review Co-authored-by: Nicola Soranzo --- doc/source/dev/api_guidelines.rst | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/source/dev/api_guidelines.rst b/doc/source/dev/api_guidelines.rst index fe3b027f4b6..6e1c822b46a 100644 --- a/doc/source/dev/api_guidelines.rst +++ b/doc/source/dev/api_guidelines.rst @@ -2,30 +2,30 @@ API Design Guidelines ===================== The following section outlines guidelines related to extending and/or modifying -the Galaxy API. The Galaxy API has grown in an ad-hoc fashion over time by -many contributors and so clients SHOULD NOT expect the API will conform to +the Galaxy API. The Galaxy API has been developed over time in an ad-hoc fashion and by +many contributors, so clients SHOULD NOT expect the API will conform to these guidelines - but developers contributing to the Galaxy API SHOULD follow these guidelines. Before Getting Started ~~~~~~~~~~~~~~~~~~~~~~ -Developers should familiarize themselves with HTTP method definitions -https://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html#sec9. While Galaxy +Developers should familiarize themselves with `HTTP method definitions +`__. While Galaxy follows these imperfectly - a serious effort should be made to use HTTP methods (GET, POST, PUT, DELETE, etc.) correctly and consistently. More information about these methods with a focus on RESTful APIs can be found at https://restfulapi.net/http-methods/. -Developers should familiarize themselves with the HTTP status code definitions -http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html. The API responses +Developers should also familiarize themselves with the `HTTP status code definitions +`__. The API responses should properly set the status code according to the result - in particular 2XX responses should be used for successful requests, 4XX for various kinds of client errors, and 5XX for the errors on the server side. -Developers should also familiarize themselves with RESTful API design -https://docs.microsoft.com/en-us/azure/architecture/best-practices/api-design. -Again Galaxy follows generic RESTful guidelines imperfectly - but a serious +Finally, developers should familiarize themselves with `RESTful API design +`__. +Again, Galaxy follows generic RESTful guidelines imperfectly, but a serious effort should be made to design the API as RESTfully and consistently as possible. Documenting the API @@ -44,7 +44,7 @@ of the static checking that can be done for the Galaxy backend. To further document your API and allow other developers to see how it functions, new API functionality should include functional tests. These functional tests should be implemented in Python and added to either the API test suite -or the integration test suite. Checkout the :ref:`Writing Tests for Galaxy <./writing_tests.md>` +or the integration test suite. Checkout the :doc:`Writing Tests for Galaxy ` documentation for more information on developing tests for Galaxy. .. note:: @@ -93,7 +93,7 @@ Backward Compatibility Backward compatibility is important and should be maintained when possible. If changing behavior in a non-backward compatible way please ensure one of the following holds - there is a strong reason to believe no consumers -depend on a behavior, the behavior is effectively broken, or the API +depend on the old behavior, the behavior is effectively broken, or the API method being modified has not been part of a tagged dist release. If these conditions do not hold, please help the Galaxy release manager ensure information about the breaking API is highlighted in the Galaxy release @@ -118,7 +118,7 @@ possible. Most of the logic should be pushed into components that are decoupled from the actual web application and the web framework. The role of the controller should be simply to adapt the web request (tied to the web framework) to simple objects (plain Python objects, -Pydantic_ models, or SQL Alchemy models), send these objects to a +Pydantic_ models, or SQLAlchemy models), send these objects to a Python component lower in the stack, and finally to translate the response to a web response. This design will ensure that Galaxy application logic is usable in as many contexts as possible and From 42bd3b269e7c5264d76c0857eb9a886a6c1d8ce0 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 13 Jan 2021 18:42:44 -0500 Subject: [PATCH 104/208] Remove unused, deprecated ControllerUnavailable. --- lib/galaxy/webapps/base/controller.py | 8 -------- lib/galaxy/webapps/base/webapp.py | 14 ++------------ 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/lib/galaxy/webapps/base/controller.py b/lib/galaxy/webapps/base/controller.py index aab9eddcf33..7bd63744154 100644 --- a/lib/galaxy/webapps/base/controller.py +++ b/lib/galaxy/webapps/base/controller.py @@ -1669,14 +1669,6 @@ class UsesExtendedMetadataMixin(SharableItemSecurityMixin): yield prefix, ("%s" % (meta)).encode("utf8", errors='replace') -class ControllerUnavailable(Exception): - """ - Deprecated: `BaseController` used to be available under the name `Root` - """ - -# ---- Utility methods ------------------------------------------------------- - - def sort_by_attr(seq, attr): """ Sort the sequence of objects by object's attribute diff --git a/lib/galaxy/webapps/base/webapp.py b/lib/galaxy/webapps/base/webapp.py index 269d10135f3..70a5db1a61b 100644 --- a/lib/galaxy/webapps/base/webapp.py +++ b/lib/galaxy/webapps/base/webapp.py @@ -124,18 +124,13 @@ class WebApplication(base.WebApplication): them to the webapp. """ from galaxy.webapps.base.controller import BaseUIController - from galaxy.webapps.base.controller import ControllerUnavailable package = import_module(package_name) controller_dir = package.__path__[0] for fname in os.listdir(controller_dir): if not(fname.startswith("_")) and fname.endswith(".py"): name = fname[:-3] module_name = package_name + "." + name - try: - module = import_module(module_name) - except ControllerUnavailable as exc: - log.debug("%s could not be loaded: %s", module_name, unicodify(exc)) - continue + module = import_module(module_name) # Look for a controller inside the modules for key in dir(module): T = getattr(module, key) @@ -149,18 +144,13 @@ class WebApplication(base.WebApplication): them to the webapp. """ from galaxy.webapps.base.controller import BaseAPIController - from galaxy.webapps.base.controller import ControllerUnavailable package = import_module(package_name) controller_dir = package.__path__[0] for fname in os.listdir(controller_dir): if not(fname.startswith("_")) and fname.endswith(".py"): name = fname[:-3] module_name = package_name + "." + name - try: - module = import_module(module_name) - except ControllerUnavailable as exc: - log.debug("%s could not be loaded: %s", module_name, unicodify(exc)) - continue + module = import_module(module_name) for key in dir(module): T = getattr(module, key) # Exclude classes such as BaseAPIController and BaseTagItemsController From 37f8684582f4efb61913767501e4e4f914b45437 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 13 Jan 2021 18:48:33 -0500 Subject: [PATCH 105/208] Extract code for walking controller modules. --- lib/galaxy/web/framework/base.py | 14 +++++++++- lib/galaxy/webapps/base/webapp.py | 45 +++++++++++-------------------- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/lib/galaxy/web/framework/base.py b/lib/galaxy/web/framework/base.py index 461a7ea7f41..d29fddf73c1 100644 --- a/lib/galaxy/web/framework/base.py +++ b/lib/galaxy/web/framework/base.py @@ -3,13 +3,14 @@ A simple WSGI application/framework. """ import io import logging -import os.path +import os import socket import tarfile import tempfile import time import types from http.cookies import CookieError, SimpleCookie +from importlib import import_module import routes import webob.compat @@ -520,3 +521,14 @@ def flatten(seq): yield smart_str(y) else: yield smart_str(x) + + +def walk_controller_modules(package_name): + package = import_module(package_name) + controller_dir = package.__path__[0] + for fname in os.listdir(controller_dir): + if not(fname.startswith("_")) and fname.endswith(".py"): + name = fname[:-3] + module_name = package_name + "." + name + module = import_module(module_name) + yield name, module diff --git a/lib/galaxy/webapps/base/webapp.py b/lib/galaxy/webapps/base/webapp.py index 70a5db1a61b..d665872af19 100644 --- a/lib/galaxy/webapps/base/webapp.py +++ b/lib/galaxy/webapps/base/webapp.py @@ -9,7 +9,6 @@ import socket import string import time from http.cookies import CookieError -from importlib import import_module from urllib.parse import urlparse import mako.lookup @@ -124,19 +123,13 @@ class WebApplication(base.WebApplication): them to the webapp. """ from galaxy.webapps.base.controller import BaseUIController - package = import_module(package_name) - controller_dir = package.__path__[0] - for fname in os.listdir(controller_dir): - if not(fname.startswith("_")) and fname.endswith(".py"): - name = fname[:-3] - module_name = package_name + "." + name - module = import_module(module_name) - # Look for a controller inside the modules - for key in dir(module): - T = getattr(module, key) - if inspect.isclass(T) and T is not BaseUIController and issubclass(T, BaseUIController): - controller = self._instantiate_controller(T, app) - self.add_ui_controller(name, controller) + for name, module in base.walk_controller_modules(package_name): + # Look for a controller inside the modules + for key in dir(module): + T = getattr(module, key) + if inspect.isclass(T) and T is not BaseUIController and issubclass(T, BaseUIController): + controller = self._instantiate_controller(T, app) + self.add_ui_controller(name, controller) def add_api_controllers(self, package_name, app): """ @@ -144,21 +137,15 @@ class WebApplication(base.WebApplication): them to the webapp. """ from galaxy.webapps.base.controller import BaseAPIController - package = import_module(package_name) - controller_dir = package.__path__[0] - for fname in os.listdir(controller_dir): - if not(fname.startswith("_")) and fname.endswith(".py"): - name = fname[:-3] - module_name = package_name + "." + name - module = import_module(module_name) - for key in dir(module): - T = getattr(module, key) - # Exclude classes such as BaseAPIController and BaseTagItemsController - if inspect.isclass(T) and not key.startswith("Base") and issubclass(T, BaseAPIController): - # By default use module_name, but allow controller to override name - controller_name = getattr(T, "controller_name", name) - controller = self._instantiate_controller(T, app) - self.add_api_controller(controller_name, controller) + for name, module in base.walk_controller_modules(package_name): + for key in dir(module): + T = getattr(module, key) + # Exclude classes such as BaseAPIController and BaseTagItemsController + if inspect.isclass(T) and not key.startswith("Base") and issubclass(T, BaseAPIController): + # By default use module_name, but allow controller to override name + controller_name = getattr(T, "controller_name", name) + controller = self._instantiate_controller(T, app) + self.add_api_controller(controller_name, controller) def _instantiate_controller(self, T, app): """ Extension point, allow apps to construct controllers differently, From a43e9820edc8cfa3a5d9479fe7029a3d648beb09 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 13 Jan 2021 18:54:30 -0500 Subject: [PATCH 106/208] Auto-discover fastapi controllers. --- lib/galaxy/webapps/galaxy/fast_app.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/fast_app.py b/lib/galaxy/webapps/galaxy/fast_app.py index 5e46e6f88c6..658c981818a 100644 --- a/lib/galaxy/webapps/galaxy/fast_app.py +++ b/lib/galaxy/webapps/galaxy/fast_app.py @@ -5,6 +5,7 @@ from fastapi.responses import JSONResponse from starlette.responses import Response from galaxy.exceptions import MessageException +from galaxy.web.framework.base import walk_controller_modules from galaxy.web.framework.decorators import ( api_error_message, validation_error_to_message_exception @@ -54,19 +55,9 @@ def initialize_fast_app(gx_app): add_exception_handler(app) wsgi_handler = WSGIMiddleware(gx_app) - from galaxy.webapps.galaxy.api import ( - datatypes, - job_lock, - jobs, - licenses, - roles, - tours, - ) - app.include_router(datatypes.router) - app.include_router(jobs.router) - app.include_router(job_lock.router) - app.include_router(licenses.router) - app.include_router(roles.router) - app.include_router(tours.router) + for _, module in walk_controller_modules('galaxy.webapps.galaxy.api'): + router = getattr(module, "router", None) + if router: + app.include_router(router) app.mount('/', wsgi_handler) return app From 8c7d74cb10d2899bbce4779d11f65c9c915d4a43 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 13 Jan 2021 18:55:43 -0500 Subject: [PATCH 107/208] Make all fastapi methods async (for consistency). --- lib/galaxy/webapps/galaxy/api/job_lock.py | 4 ++-- lib/galaxy/webapps/galaxy/api/jobs.py | 2 +- lib/galaxy/webapps/galaxy/api/roles.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/job_lock.py b/lib/galaxy/webapps/galaxy/api/job_lock.py index f64e8809c8a..cf8482a6cd3 100644 --- a/lib/galaxy/webapps/galaxy/api/job_lock.py +++ b/lib/galaxy/webapps/galaxy/api/job_lock.py @@ -11,12 +11,12 @@ router = APIRouter(tags=['job_lock']) @router.get('/api/job_lock') -def job_lock_status(job_manager=Depends(get_job_manager), admin_user=Depends(get_admin_user)) -> JobLock: +async def job_lock_status(job_manager=Depends(get_job_manager), admin_user=Depends(get_admin_user)) -> JobLock: """Get job lock status.""" return job_manager.job_lock() @router.put('/api/job_lock') -def update_job_lock(job_manager=Depends(get_job_manager), admin_user=Depends(get_admin_user), job_lock: JobLock = Body(...)) -> JobLock: +async def update_job_lock(job_manager=Depends(get_job_manager), admin_user=Depends(get_admin_user), job_lock: JobLock = Body(...)) -> JobLock: """Set job lock status.""" return job_manager.update_job_lock(job_lock) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index a6996f9d88c..fe866a1ae94 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -70,7 +70,7 @@ class FastAPIJobs: hda_manager: hdas.HDAManager = Depends(get_hda_manager) @router.get("/api/job/{id}") - def show(self, id: EncodedDatabaseIdField, trans: ProvidesUserContext = Depends(get_trans), full: typing.Optional[bool] = False) -> typing.Dict: + async def show(self, id: EncodedDatabaseIdField, trans: ProvidesUserContext = Depends(get_trans), full: typing.Optional[bool] = False) -> typing.Dict: """ Return dictionary containing description of job data diff --git a/lib/galaxy/webapps/galaxy/api/roles.py b/lib/galaxy/webapps/galaxy/api/roles.py index 0414bd39116..37cfb8aa564 100644 --- a/lib/galaxy/webapps/galaxy/api/roles.py +++ b/lib/galaxy/webapps/galaxy/api/roles.py @@ -63,18 +63,18 @@ class FastAPIRoles: role_manager: RoleManager = Depends(get_role_manager) @router.get('/api/roles') - def index(self, trans: ProvidesUserContext = Depends(get_trans)) -> RoleListModel: + async def index(self, trans: ProvidesUserContext = Depends(get_trans)) -> RoleListModel: roles = self.role_manager.list_displayable_roles(trans) return RoleListModel(__root__=[role_to_model(trans, r) for r in roles]) @router.get('/api/roles/{id}') - def show(self, id: EncodedDatabaseIdField, trans: ProvidesUserContext = Depends(get_trans)) -> RoleModel: + async def show(self, id: EncodedDatabaseIdField, trans: ProvidesUserContext = Depends(get_trans)) -> RoleModel: role_id = trans.app.security.decode_id(id) role = self.role_manager.get(trans, role_id) return role_to_model(trans, role) @router.post("/api/roles") - def create(self, trans: ProvidesUserContext = Depends(get_trans), admin_user=Depends(get_admin_user), role_definition_model: RoleDefinitionModel = Body(...)) -> RoleModel: + async def create(self, trans: ProvidesUserContext = Depends(get_trans), admin_user=Depends(get_admin_user), role_definition_model: RoleDefinitionModel = Body(...)) -> RoleModel: role = self.role_manager.create_role(trans, role_definition_model) return role_to_model(trans, role) From 5543ddf7287f44ac6a77329a0323106553ac0722 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 14 Jan 2021 06:45:20 -0500 Subject: [PATCH 108/208] Revert "Make all fastapi methods async (for consistency)." This reverts commit 8c7d74cb10d2899bbce4779d11f65c9c915d4a43. --- lib/galaxy/webapps/galaxy/api/job_lock.py | 4 ++-- lib/galaxy/webapps/galaxy/api/jobs.py | 2 +- lib/galaxy/webapps/galaxy/api/roles.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/job_lock.py b/lib/galaxy/webapps/galaxy/api/job_lock.py index cf8482a6cd3..f64e8809c8a 100644 --- a/lib/galaxy/webapps/galaxy/api/job_lock.py +++ b/lib/galaxy/webapps/galaxy/api/job_lock.py @@ -11,12 +11,12 @@ router = APIRouter(tags=['job_lock']) @router.get('/api/job_lock') -async def job_lock_status(job_manager=Depends(get_job_manager), admin_user=Depends(get_admin_user)) -> JobLock: +def job_lock_status(job_manager=Depends(get_job_manager), admin_user=Depends(get_admin_user)) -> JobLock: """Get job lock status.""" return job_manager.job_lock() @router.put('/api/job_lock') -async def update_job_lock(job_manager=Depends(get_job_manager), admin_user=Depends(get_admin_user), job_lock: JobLock = Body(...)) -> JobLock: +def update_job_lock(job_manager=Depends(get_job_manager), admin_user=Depends(get_admin_user), job_lock: JobLock = Body(...)) -> JobLock: """Set job lock status.""" return job_manager.update_job_lock(job_lock) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index fe866a1ae94..a6996f9d88c 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -70,7 +70,7 @@ class FastAPIJobs: hda_manager: hdas.HDAManager = Depends(get_hda_manager) @router.get("/api/job/{id}") - async def show(self, id: EncodedDatabaseIdField, trans: ProvidesUserContext = Depends(get_trans), full: typing.Optional[bool] = False) -> typing.Dict: + def show(self, id: EncodedDatabaseIdField, trans: ProvidesUserContext = Depends(get_trans), full: typing.Optional[bool] = False) -> typing.Dict: """ Return dictionary containing description of job data diff --git a/lib/galaxy/webapps/galaxy/api/roles.py b/lib/galaxy/webapps/galaxy/api/roles.py index 37cfb8aa564..0414bd39116 100644 --- a/lib/galaxy/webapps/galaxy/api/roles.py +++ b/lib/galaxy/webapps/galaxy/api/roles.py @@ -63,18 +63,18 @@ class FastAPIRoles: role_manager: RoleManager = Depends(get_role_manager) @router.get('/api/roles') - async def index(self, trans: ProvidesUserContext = Depends(get_trans)) -> RoleListModel: + def index(self, trans: ProvidesUserContext = Depends(get_trans)) -> RoleListModel: roles = self.role_manager.list_displayable_roles(trans) return RoleListModel(__root__=[role_to_model(trans, r) for r in roles]) @router.get('/api/roles/{id}') - async def show(self, id: EncodedDatabaseIdField, trans: ProvidesUserContext = Depends(get_trans)) -> RoleModel: + def show(self, id: EncodedDatabaseIdField, trans: ProvidesUserContext = Depends(get_trans)) -> RoleModel: role_id = trans.app.security.decode_id(id) role = self.role_manager.get(trans, role_id) return role_to_model(trans, role) @router.post("/api/roles") - async def create(self, trans: ProvidesUserContext = Depends(get_trans), admin_user=Depends(get_admin_user), role_definition_model: RoleDefinitionModel = Body(...)) -> RoleModel: + def create(self, trans: ProvidesUserContext = Depends(get_trans), admin_user=Depends(get_admin_user), role_definition_model: RoleDefinitionModel = Body(...)) -> RoleModel: role = self.role_manager.create_role(trans, role_definition_model) return role_to_model(trans, role) From 970edcd8c3822b6e5eba58656525df6aedd5ed63 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 14 Jan 2021 15:30:28 +0100 Subject: [PATCH 109/208] Ignore mypy error with Union in response_model --- lib/galaxy/webapps/galaxy/api/datatypes.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/datatypes.py b/lib/galaxy/webapps/galaxy/api/datatypes.py index 55d748b886b..ea757ae1138 100644 --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -61,12 +61,11 @@ def get_datatypes_registry(app: UniverseApplication = Depends(get_app)) -> Regis class FastAPIDatatypes: datatypes_registry: Registry = Depends(get_datatypes_registry) - @router.get( - '/api/datatypes', - summary="Lists all available data types", - response_model=Union[List[DatatypeDetails], List[str]], - response_description="List of data types", - ) + # mypy does not like the use of Union in the response_model but it must be used to support + # the two possible types of response for this endpoint. + # Please see: https://stackoverflow.com/questions/62264787/mypy-fastapi-response-model + # Aparently this may be related to the Python version + @router.get('/api/datatypes', summary="Lists all available data types", response_model=Union[List[DatatypeDetails], List[str]], response_description="List of data types") # type: ignore async def index( self, extension_only: Optional[bool] = ExtensionOnlyQueryParam, From 40a7cd8f3c4dcfcbb0887aefa5d2714ad4b6b2d0 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 14 Jan 2021 16:11:11 +0100 Subject: [PATCH 110/208] Don't attempt to use docker cli if it is not available This fixes https://github.com/galaxyproject/galaxy/issues/11125, which became a problem with https://github.com/galaxyproject/galaxy/pull/10814 and the recent removal of the docker runtime from kubernetes. This is the quick fix, but I don't like the guessing and disabling that we do if docker is not available or not functional. I'm hesitating between adding a config option for the resolver (`docker_cli_available` or `docker_list_images` + `docker_pull_images`) and adding a `KubernetesContainerResolver` / `RemoteContainerResolver` class that doesn't assume docker is available for listing and pulling iages. Not wanting to list/pull images on the Galaxy node is probably a reasonable thing outside of kubernetes as well, so I tend to go with a `RemoteContainerResolver` class in dev ? --- .../deps/container_resolvers/mulled.py | 90 ++++++++++++------- packages/tool_util/test-requirements.txt | 1 + .../tool_util/test_container_resolution.py | 53 +++++++++++ 3 files changed, 112 insertions(+), 32 deletions(-) create mode 100644 test/unit/tool_util/test_container_resolution.py diff --git a/lib/galaxy/tool_util/deps/container_resolvers/mulled.py b/lib/galaxy/tool_util/deps/container_resolvers/mulled.py index 2b01e04a2f8..e706ce10ae9 100644 --- a/lib/galaxy/tool_util/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tool_util/deps/container_resolvers/mulled.py @@ -9,6 +9,7 @@ import subprocess from galaxy.util import ( string_as_bool, unicodify, + which, ) from galaxy.util.commands import shell from ..container_classes import CONTAINER_CLASSES @@ -322,7 +323,24 @@ def targets_to_mulled_name(targets, hash_func, namespace, resolution_cache=None, return name -class CachedMulledDockerContainerResolver(ContainerResolver): +class DockerContainerResolver(ContainerResolver): + + def __init__(self, *args, **kwargs): + self._docker_cli_available = bool(which('docker')) + super().__init__(*args, **kwargs) + + @property + def docker_cli_available(self): + return self._docker_cli_available + + @docker_cli_available.setter + def docker_cli_available(self, value): + if not value: + log.info('Docker CLI not available, cannot list or pull images in Galaxy process. Does not impact kubernetes.') + self._docker_cli_available = value + + +class CachedMulledDockerContainerResolver(DockerContainerResolver): resolver_type = "cached_mulled" container_type = "docker" @@ -334,7 +352,7 @@ class CachedMulledDockerContainerResolver(ContainerResolver): self.hash_func = hash_func def resolve(self, enabled_container_types, tool_info, **kwds): - if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: + if not self.docker_cli_available or tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: return None targets = mulled_targets(tool_info) @@ -367,7 +385,7 @@ class CachedMulledSingularityContainerResolver(ContainerResolver): return "CachedMulledSingularityContainerResolver[cache_directory=%s]" % self.cache_directory -class MulledDockerContainerResolver(ContainerResolver): +class MulledDockerContainerResolver(DockerContainerResolver): """Look for mulled images matching tool dependencies.""" resolver_type = "mulled" @@ -382,11 +400,18 @@ class MulledDockerContainerResolver(ContainerResolver): self.auto_install = string_as_bool(auto_install) def cached_container_description(self, targets, namespace, hash_func, resolution_cache): - return docker_cached_container_description(targets, namespace, hash_func, resolution_cache) + try: + return docker_cached_container_description(targets, namespace, hash_func, resolution_cache) + except subprocess.CalledProcessError: + # We should only get here if a docker binary is available, but command quits with a non-zero exit code, + # e.g if the docker daemon is not available + log.exception('An error occured while listing cached docker images. Disabling docker image cache.') + self.docker_cli_available = False def pull(self, container): - command = container.build_pull_command() - shell(command) + if self.docker_cli_available: + command = container.build_pull_command() + shell(command) def resolve(self, enabled_container_types, tool_info, install=False, session=None, **kwds): resolution_cache = kwds.get("resolution_cache") @@ -407,30 +432,31 @@ class MulledDockerContainerResolver(ContainerResolver): type=self.container_type, shell=self.shell, ) - if install and not self.cached_container_description( - targets, - namespace=self.namespace, - hash_func=self.hash_func, - resolution_cache=resolution_cache, - ): - destination_info = {} - destination_for_container_type = kwds.get('destination_for_container_type') - if destination_for_container_type: - destination_info = destination_for_container_type(self.container_type) - container = CONTAINER_CLASSES[self.container_type](container_description.identifier, - self.app_info, - tool_info, - destination_info, - {}, - container_description) - self.pull(container) - if not self.auto_install: - container_description = self.cached_container_description( - targets, - namespace=self.namespace, - hash_func=self.hash_func, - resolution_cache=resolution_cache, - ) + if self.docker_cli_available: + if install and not self.cached_container_description( + targets, + namespace=self.namespace, + hash_func=self.hash_func, + resolution_cache=resolution_cache, + ): + destination_info = {} + destination_for_container_type = kwds.get('destination_for_container_type') + if destination_for_container_type: + destination_info = destination_for_container_type(self.container_type) + container = CONTAINER_CLASSES[self.container_type](container_description.identifier, + self.app_info, + tool_info, + destination_info, + {}, + container_description) + self.pull(container) + if not self.auto_install: + container_description = self.cached_container_description( + targets, + namespace=self.namespace, + hash_func=self.hash_func, + resolution_cache=resolution_cache, + ) return container_description def __str__(self): @@ -463,7 +489,7 @@ class MulledSingularityContainerResolver(MulledDockerContainerResolver): return "MulledSingularityContainerResolver[namespace=%s]" % self.namespace -class BuildMulledDockerContainerResolver(ContainerResolver): +class BuildMulledDockerContainerResolver(DockerContainerResolver): """Build for Docker mulled images matching tool dependencies.""" resolver_type = "build_mulled" @@ -488,7 +514,7 @@ class BuildMulledDockerContainerResolver(ContainerResolver): self.auto_init = self._get_config_option("involucro_auto_init", True) def resolve(self, enabled_container_types, tool_info, install=False, **kwds): - if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: + if not self.docker_cli_available or tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: return None targets = mulled_targets(tool_info) diff --git a/packages/tool_util/test-requirements.txt b/packages/tool_util/test-requirements.txt index e079f8a6038..12136494321 100644 --- a/packages/tool_util/test-requirements.txt +++ b/packages/tool_util/test-requirements.txt @@ -1 +1,2 @@ pytest +pytest-mock diff --git a/test/unit/tool_util/test_container_resolution.py b/test/unit/tool_util/test_container_resolution.py new file mode 100644 index 00000000000..e355929c966 --- /dev/null +++ b/test/unit/tool_util/test_container_resolution.py @@ -0,0 +1,53 @@ +from subprocess import CalledProcessError + +from galaxy.tool_util.deps.container_resolvers.mulled import ( + CachedMulledDockerContainerResolver, + MulledDockerContainerResolver, +) +from galaxy.tool_util.deps.dependencies import ToolInfo +from galaxy.tool_util.deps.requirements import ToolRequirement + + +def test_docker_container_resolver_detects_docker_cli_absent(mocker): + mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.which', return_value=None) + resolver = CachedMulledDockerContainerResolver() + assert resolver.docker_cli_available is False + + +def test_docker_container_resolver_detects_docker_cli(mocker): + mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled', return_value='/bin/docker') + resolver = CachedMulledDockerContainerResolver() + assert resolver.docker_cli_available + + +def test_cached_docker_container_docker_cli_absent_resolve(mocker): + mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.which', return_value=None) + resolver = CachedMulledDockerContainerResolver() + assert resolver.docker_cli_available is False + assert resolver.resolve(enabled_container_types=[], tool_info={}) is None + + +def test_docker_container_docker_cli_absent_resolve(mocker): + mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.which', return_value=None) + resolver = MulledDockerContainerResolver() + assert resolver.docker_cli_available is False + requirement = ToolRequirement(name="samtools", version="1.10", type="package") + tool_info = ToolInfo(requirements=[requirement]) + mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.targets_to_mulled_name', return_value='samtools:1.10--h2e538c0_3') + container_description = resolver.resolve(enabled_container_types=['docker'], tool_info=tool_info) + assert container_description.type == 'docker' + assert container_description.identifier == 'quay.io/biocontainers/samtools:1.10--h2e538c0_3' + + +def test_docker_container_docker_cli_exception_resolve(mocker): + mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.which', return_value='/bin/docker') + resolver = MulledDockerContainerResolver() + assert resolver.docker_cli_available is True + requirement = ToolRequirement(name="samtools", version="1.10", type="package") + tool_info = ToolInfo(requirements=[requirement]) + mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.targets_to_mulled_name', return_value='samtools:1.10--h2e538c0_3') + mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.docker_cached_container_description', side_effect=CalledProcessError(1, 'bla')) + container_description = resolver.resolve(enabled_container_types=['docker'], tool_info=tool_info, install=True) + assert resolver.docker_cli_available is False + assert container_description.type == 'docker' + assert container_description.identifier == 'quay.io/biocontainers/samtools:1.10--h2e538c0_3' From 42ff20f6a00729dee1ead039c75495731af142ac Mon Sep 17 00:00:00 2001 From: selten Date: Thu, 14 Jan 2021 17:43:35 +0100 Subject: [PATCH 111/208] Minor spelling error fix Fixes a minor spelling error. --- lib/galaxy/tool_util/verify/interactor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/verify/interactor.py b/lib/galaxy/tool_util/verify/interactor.py index 0c110414a78..0b7791110d5 100644 --- a/lib/galaxy/tool_util/verify/interactor.py +++ b/lib/galaxy/tool_util/verify/interactor.py @@ -562,7 +562,7 @@ class GalaxyInteractorApi: print("| Dataset Info:") print(self.format_for_summary(dataset_info.get("misc_info", ""), "Dataset info is empty.")) print("| Peek:") - print(self.format_for_summary(dataset_info.get("peek", ""), "Peek unavilable.")) + print(self.format_for_summary(dataset_info.get("peek", ""), "Peek unavailable.")) except Exception: print("| *TEST FRAMEWORK ERROR FETCHING DATASET DETAILS*") try: From f11d675753680506275ea9410aa0cdde7a705745 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 14 Jan 2021 18:06:42 +0100 Subject: [PATCH 112/208] Use InferringRouter in datatypes router And remove all response_model declarations --- lib/galaxy/webapps/galaxy/api/datatypes.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/datatypes.py b/lib/galaxy/webapps/galaxy/api/datatypes.py index ea757ae1138..e351ed521e7 100644 --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -13,8 +13,8 @@ from fastapi import ( Depends, Query, ) -from fastapi.routing import APIRouter from fastapi_utils.cbv import cbv +from fastapi_utils.inferring_router import InferringRouter as APIRouter from galaxy.app import UniverseApplication from galaxy.datatypes.registry import Registry @@ -61,11 +61,11 @@ def get_datatypes_registry(app: UniverseApplication = Depends(get_app)) -> Regis class FastAPIDatatypes: datatypes_registry: Registry = Depends(get_datatypes_registry) - # mypy does not like the use of Union in the response_model but it must be used to support - # the two possible types of response for this endpoint. - # Please see: https://stackoverflow.com/questions/62264787/mypy-fastapi-response-model - # Aparently this may be related to the Python version - @router.get('/api/datatypes', summary="Lists all available data types", response_model=Union[List[DatatypeDetails], List[str]], response_description="List of data types") # type: ignore + @router.get( + '/api/datatypes', + summary="Lists all available data types", + response_description="List of data types", + ) async def index( self, extension_only: Optional[bool] = ExtensionOnlyQueryParam, @@ -77,7 +77,6 @@ class FastAPIDatatypes: @router.get( '/api/datatypes/mapping', summary="Returns mappings for data types and their implementing classes", - response_model=DatatypesMap, response_description="Dictionary to map data types with their classes" ) async def mapping(self) -> DatatypesMap: @@ -87,7 +86,6 @@ class FastAPIDatatypes: @router.get( '/api/datatypes/types_and_mapping', summary="Returns all the data types extensions and their mappings", - response_model=DatatypesCombinedMap, response_description="Dictionary to map data types with their classes" ) async def types_and_mapping( @@ -106,7 +104,6 @@ class FastAPIDatatypes: @router.get( '/api/datatypes/sniffers', summary="Returns the list of all installed sniffers", - response_model=List[str], response_description="List of datatype sniffers" ) async def sniffers(self) -> List[str]: @@ -116,7 +113,6 @@ class FastAPIDatatypes: @router.get( '/api/datatypes/converters', summary="Returns the list of all installed converters", - response_model=DatatypeConverterList, response_description="List of all datatype converters" ) async def converters(self) -> DatatypeConverterList: @@ -126,7 +122,6 @@ class FastAPIDatatypes: @router.get( '/api/datatypes/edam_formats', summary="Returns a dictionary/map of datatypes and EDAM formats", - response_model=Dict[str, str], response_description="Dictionary/map of datatypes and EDAM formats" ) async def edam_formats(self) -> Dict[str, str]: @@ -136,7 +131,6 @@ class FastAPIDatatypes: @router.get( '/api/datatypes/edam_data', summary="Returns a dictionary/map of datatypes and EDAM data", - response_model=Dict[str, str], response_description="Dictionary/map of datatypes and EDAM data" ) async def edam_data(self) -> Dict[str, str]: From e3f7948a4aea8bc254774459341bf1cd39745c68 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 14 Jan 2021 18:09:22 +0100 Subject: [PATCH 113/208] Use InferringRouter in licenses router And remove all response_model declarations --- lib/galaxy/webapps/galaxy/api/licenses.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/licenses.py b/lib/galaxy/webapps/galaxy/api/licenses.py index c058acad185..e7e6f09df93 100644 --- a/lib/galaxy/webapps/galaxy/api/licenses.py +++ b/lib/galaxy/webapps/galaxy/api/licenses.py @@ -4,8 +4,8 @@ from fastapi import ( Depends, Path ) -from fastapi.routing import APIRouter from fastapi_utils.cbv import cbv +from fastapi_utils.inferring_router import InferringRouter as APIRouter from galaxy.managers.licenses import ( LicenseMetadataModel, @@ -34,7 +34,6 @@ class FastAPILicenses: @router.get('/api/licenses', summary="Lists all available SPDX licenses", - response_model=List[LicenseMetadataModel], response_description="List of SPDX licenses") async def index(self) -> List[LicenseMetadataModel]: """Returns an index with all the available [SPDX licenses](https://spdx.org/licenses/).""" @@ -42,7 +41,6 @@ class FastAPILicenses: @router.get('/api/licenses/{id}', summary="Gets the SPDX license metadata associated with the short identifier", - response_model=LicenseMetadataModel, response_description="SPDX license metadata") async def get(self, id=LicenseIdPath) -> LicenseMetadataModel: """Returns the license metadata associated with the given From 5fed2b4c791f498a88cab36a0e82cd6db431a1fa Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 14 Jan 2021 18:18:06 +0100 Subject: [PATCH 114/208] Don't load BuildMulled resolvers if docker not available --- .../tool_util/deps/container_resolvers/mulled.py | 2 +- lib/galaxy/tool_util/deps/containers.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tool_util/deps/container_resolvers/mulled.py b/lib/galaxy/tool_util/deps/container_resolvers/mulled.py index e706ce10ae9..5be8c994afa 100644 --- a/lib/galaxy/tool_util/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tool_util/deps/container_resolvers/mulled.py @@ -514,7 +514,7 @@ class BuildMulledDockerContainerResolver(DockerContainerResolver): self.auto_init = self._get_config_option("involucro_auto_init", True) def resolve(self, enabled_container_types, tool_info, install=False, **kwds): - if not self.docker_cli_available or tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: + if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: return None targets = mulled_targets(tool_info) diff --git a/lib/galaxy/tool_util/deps/containers.py b/lib/galaxy/tool_util/deps/containers.py index 7107608e6fe..af9778c122a 100644 --- a/lib/galaxy/tool_util/deps/containers.py +++ b/lib/galaxy/tool_util/deps/containers.py @@ -222,9 +222,15 @@ class ContainerRegistry: CachedMulledSingularityContainerResolver(self.app_info, namespace="local"), MulledDockerContainerResolver(self.app_info, namespace="biocontainers"), MulledSingularityContainerResolver(self.app_info, namespace="biocontainers"), - BuildMulledDockerContainerResolver(self.app_info), - BuildMulledSingularityContainerResolver(self.app_info), ]) + # BuildMulledDockerContainerResolver and BuildMulledSingularityContainerResolver both need the docker daemon to build images. + # If docker is not available, we don't load them. + build_mulled_docker_container_resolver = BuildMulledDockerContainerResolver(self.app_info) + if build_mulled_docker_container_resolver.docker_cli_available: + default_resolvers.extend([ + build_mulled_docker_container_resolver, + BuildMulledSingularityContainerResolver(self.app_info), + ]) return default_resolvers def __resolvers_dict(self): From b358e0a97306eafcffd0105bb4f823499c35ab66 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Thu, 14 Jan 2021 18:02:47 +0100 Subject: [PATCH 115/208] fix #10982 --- .../components/LibraryFolder/TopToolbar/FolderTopBar.vue | 6 +++--- .../src/components/LibraryFolder/TopToolbar/add-datasets.js | 1 + lib/galaxy/managers/configuration.py | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/client/src/components/LibraryFolder/TopToolbar/FolderTopBar.vue b/client/src/components/LibraryFolder/TopToolbar/FolderTopBar.vue index afb005d62e8..cff314c322d 100644 --- a/client/src/components/LibraryFolder/TopToolbar/FolderTopBar.vue +++ b/client/src/components/LibraryFolder/TopToolbar/FolderTopBar.vue @@ -53,7 +53,7 @@ from History @@ -209,7 +209,7 @@ export default { data() { return { is_admin: false, - user_library_import_dir: false, + user_library_import_dir_available: false, library_import_dir: false, allow_library_path_paste: false, list_genomes: [], @@ -230,7 +230,7 @@ export default { created() { const Galaxy = getGalaxyInstance(); this.is_admin = Galaxy.user.attributes.is_admin; - this.user_library_import_dir = Galaxy.config.user_library_import_dir; + this.user_library_import_dir_available = Galaxy.config.user_library_import_dir_available; this.library_import_dir = Galaxy.config.library_import_dir; this.allow_library_path_paste = Galaxy.config.allow_library_path_paste; diff --git a/client/src/components/LibraryFolder/TopToolbar/add-datasets.js b/client/src/components/LibraryFolder/TopToolbar/add-datasets.js index 7473509bbbe..9f3a2cc977c 100644 --- a/client/src/components/LibraryFolder/TopToolbar/add-datasets.js +++ b/client/src/components/LibraryFolder/TopToolbar/add-datasets.js @@ -411,6 +411,7 @@ var AddDatasets = Backbone.View.extend({ if (typeof response.responseJSON !== "undefined") { if (response.responseJSON.err_code === 404001) { Toast.warning(response.responseJSON.err_msg); + getGalaxyInstance().modal.hide(); } else { Toast.error(response.responseJSON.err_msg); } diff --git a/lib/galaxy/managers/configuration.py b/lib/galaxy/managers/configuration.py index 4e74f03fbfb..8abed845f46 100644 --- a/lib/galaxy/managers/configuration.py +++ b/lib/galaxy/managers/configuration.py @@ -109,6 +109,7 @@ class ConfigSerializer(base.ModelSerializer): 'select_type_workflow_threshold' : _use_config, 'file_sources_configured' : lambda config, key, **context: self.app.file_sources.custom_sources_configured, 'upload_from_form_button' : _use_config, + 'user_library_import_dir_available' : lambda config, key, **context: True if config.get('user_library_import_dir') else False, } From f24ee6388d6878b228b6528e634c76102d0209bf Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Thu, 14 Jan 2021 20:30:23 +0100 Subject: [PATCH 116/208] simplify getting user_library_import_dir Co-authored-by: Marius van den Beek --- lib/galaxy/managers/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/managers/configuration.py b/lib/galaxy/managers/configuration.py index 8abed845f46..f4680b14008 100644 --- a/lib/galaxy/managers/configuration.py +++ b/lib/galaxy/managers/configuration.py @@ -109,7 +109,7 @@ class ConfigSerializer(base.ModelSerializer): 'select_type_workflow_threshold' : _use_config, 'file_sources_configured' : lambda config, key, **context: self.app.file_sources.custom_sources_configured, 'upload_from_form_button' : _use_config, - 'user_library_import_dir_available' : lambda config, key, **context: True if config.get('user_library_import_dir') else False, + 'user_library_import_dir_available' : lambda config, key, **context: bool(config.get('user_library_import_dir')), } From 1476a0b6addd3372833a12b0e45c01cde2717236 Mon Sep 17 00:00:00 2001 From: Mason Houtz Date: Thu, 14 Jan 2021 12:42:38 -0800 Subject: [PATCH 117/208] replaced global vue with test vue in unit tests --- .../DatasetInformation/DatasetInformation.test.js | 4 ++++ client/src/components/HistoryExport/Index.vue | 10 +++++----- .../ToolsView/ToolsSchemaJson/ToolsJson.test.js | 7 +++++-- .../components/Toolshed/InstalledList/Index.test.js | 4 ++++ .../RepositoryDetails/InstallationButton.test.js | 4 ++++ .../RepositoryDetails/InstallationSettings.test.js | 4 ++++ .../Toolshed/RepositoryDetails/RepositoryTools.test.js | 4 ++++ .../Toolshed/SearchList/ServerSelection.test.js | 4 ++++ client/src/components/Upload/UploadModal.test.js | 2 -- client/src/components/User/CloudAuth/CloudAuth.test.js | 5 ----- client/src/components/User/RecentInvocations.test.js | 5 ++++- .../Workflow/Editor/RefactorConfirmationModal.test.js | 5 +++-- client/src/components/Workflow/Invocations.test.js | 5 +++++ .../WorkflowInvocationState.test.js | 5 +++++ client/src/components/admin/BaseList.test.js | 4 ++++ 15 files changed, 55 insertions(+), 17 deletions(-) diff --git a/client/src/components/DatasetInformation/DatasetInformation.test.js b/client/src/components/DatasetInformation/DatasetInformation.test.js index 5fa47ff2e5f..dfc341eb83b 100644 --- a/client/src/components/DatasetInformation/DatasetInformation.test.js +++ b/client/src/components/DatasetInformation/DatasetInformation.test.js @@ -1,6 +1,7 @@ import axios from "axios"; import MockAdapter from "axios-mock-adapter"; import { mount } from "@vue/test-utils"; +import { getLocalVue } from "jest/helpers"; import DatasetInformation from "./DatasetInformation"; import datasetResponse from "./testData/datasetResponse"; import flushPromises from "flush-promises"; @@ -18,6 +19,8 @@ const mockDatasetProvider = { }, }; +const localVue = getLocalVue(); + describe("DatasetInformation/DatasetInformation.vue", () => { let wrapper; let datasetInfoTable; @@ -42,6 +45,7 @@ describe("DatasetInformation/DatasetInformation.vue", () => { stubs: { DatasetProvider: mockDatasetProvider, }, + localVue, }); datasetInfoTable = wrapper.find("#dataset-details"); await flushPromises(); diff --git a/client/src/components/HistoryExport/Index.vue b/client/src/components/HistoryExport/Index.vue index 51db0e2fccd..f8078c90ca2 100644 --- a/client/src/components/HistoryExport/Index.vue +++ b/client/src/components/HistoryExport/Index.vue @@ -25,20 +25,20 @@