diff --git a/.ci/flake8_blacklist.txt b/.ci/flake8_blacklist.txt index 26ece30af32..f5276b3fc47 100644 --- a/.ci/flake8_blacklist.txt +++ b/.ci/flake8_blacklist.txt @@ -15,3 +15,5 @@ lib/galaxy/util/jstree.py lib/galaxy/web/proxy/js/node_modules static/maps static/scripts +build +dist diff --git a/.gitignore b/.gitignore index e52c67610a4..d17e2a9d68f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,17 @@ scripts/scramble/archives .venv .venv3 +# Python build artifacts +build +dist +lib/galaxy.egg-info +.eggs + +# Generated static content +lib/galaxy/web/framework/static/maps +lib/galaxy/web/framework/static/scripts +lib/galaxy/web/framework/static/style + # Database stuff database/beaker_sessions database/citations diff --git a/.travis.yml b/.travis.yml index 32b0bba580d..942e8625b46 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ install: sh scripts/common_startup.sh wget -q https://github.com/jmchilton/galaxy-downloads/raw/master/db_gx_rev_0127.sqlite mv db_gx_rev_0127.sqlite database/universe.sqlite - sh manage_db.sh -c ./config/galaxy.yml.sample upgrade + sh manage_db.sh upgrade elif [ "$TOX_ENV" == "py35-first_startup" ]; then MINICONDA_URL="https://repo.anaconda.com/miniconda" MINICONDA_FILE="Miniconda3-latest-MacOSX-x86_64.sh" diff --git a/client/galaxy/scripts/components/PluginList.vue b/client/galaxy/scripts/components/PluginList.vue index 2909e7e9439..2f70becf0c7 100644 --- a/client/galaxy/scripts/components/PluginList.vue +++ b/client/galaxy/scripts/components/PluginList.vue @@ -2,14 +2,16 @@
{{ error }}
- +
+ +
diff --git a/client/galaxy/scripts/entry/panels/admin-panel.js b/client/galaxy/scripts/entry/panels/admin-panel.js index e67252912a6..a91e6703a8e 100644 --- a/client/galaxy/scripts/entry/panels/admin-panel.js +++ b/client/galaxy/scripts/entry/panels/admin-panel.js @@ -93,16 +93,6 @@ const AdminPanel = Backbone.View.extend({ target: "__use_router__", enabled: this.settings.is_tool_shed_installed }, - { - title: _l("Install new tools (Legacy)"), - url: "admin_toolshed/browse_tool_sheds", - enabled: this.settings.is_tool_shed_installed - }, - { - title: "Install new tools (Beta)", - url: "admin_toolshed/browse_toolsheds", - enabled: this.settings.is_tool_shed_installed && this.config.enable_beta_ts_api_install - }, { title: _l("Monitor installation"), url: "admin_toolshed/monitor_repository_installation", diff --git a/client/galaxy/scripts/entry/panels/tool-panel.js b/client/galaxy/scripts/entry/panels/tool-panel.js index d3c134a77d0..47b1ab4cfbe 100644 --- a/client/galaxy/scripts/entry/panels/tool-panel.js +++ b/client/galaxy/scripts/entry/panels/tool-panel.js @@ -141,9 +141,9 @@ const ToolPanel = Backbone.View.extend({
diff --git a/client/galaxy/scripts/mvc/collection/list-of-pairs-collection-creator.js b/client/galaxy/scripts/mvc/collection/list-of-pairs-collection-creator.js index 15ec4654c2b..52d84691d82 100644 --- a/client/galaxy/scripts/mvc/collection/list-of-pairs-collection-creator.js +++ b/client/galaxy/scripts/mvc/collection/list-of-pairs-collection-creator.js @@ -1568,7 +1568,7 @@ var PairedCollectionCreator = Backbone.View.extend(baseMVC.LoggableMixin) "", '', "
", - '
', + '
', '', @@ -1596,7 +1596,7 @@ var PairedCollectionCreator = Backbone.View.extend(baseMVC.LoggableMixin) "", '', "
", - '
', + '
', '', diff --git a/client/galaxy/scripts/mvc/library/library-dataset-view.js b/client/galaxy/scripts/mvc/library/library-dataset-view.js index 53c6d30eb6f..8137376b77f 100644 --- a/client/galaxy/scripts/mvc/library/library-dataset-view.js +++ b/client/galaxy/scripts/mvc/library/library-dataset-view.js @@ -703,6 +703,12 @@ var LibraryDatasetView = Backbone.View.extend({
<% } %> + <% if (item.get("update_time")) { %> + + + + + <% } %> <% if (item.get("date_uploaded")) { %> @@ -880,6 +886,12 @@ var LibraryDatasetView = Backbone.View.extend({ <% } %> + <% if (ldda.get("update_time")) { %> + + + + + <% } %> <% if (ldda.get("date_uploaded")) { %> @@ -1006,6 +1018,10 @@ var LibraryDatasetView = Backbone.View.extend({ + + + + diff --git a/client/galaxy/scripts/mvc/library/library-folderlist-view.js b/client/galaxy/scripts/mvc/library/library-folderlist-view.js index 1b6c271f99c..cedc161f341 100644 --- a/client/galaxy/scripts/mvc/library/library-folderlist-view.js +++ b/client/galaxy/scripts/mvc/library/library-folderlist-view.js @@ -517,7 +517,7 @@ var FolderListView = Backbone.View.extend({
<%= _.escape(item.get("file_size")) %>
Date last updated (UTC)<%= _.escape(item.get("update_time")) %>
Date uploaded (UTC)<%= _.escape(ldda.get("file_size")) %>
Date last updated (UTC)<%= _.escape(ldda.get("update_time")) %>
Date uploaded (UTC)Size <%= _.escape(item.get("file_size")) %>
Date last updated (UTC)<%= _.escape(item.get("update_time")) %>
Date uploaded (UTC) <%= _.escape(item.get("date_uploaded")) %> - Time Updated (UTC) + Date Updated (UTC) diff --git a/client/galaxy/scripts/mvc/library/library-util.js b/client/galaxy/scripts/mvc/library/library-util.js index c4d546c4b25..bb174a7cb56 100644 --- a/client/galaxy/scripts/mvc/library/library-util.js +++ b/client/galaxy/scripts/mvc/library/library-util.js @@ -1,45 +1,38 @@ /** - * Create alphabetical based two-argument comparator - * that takes into account that Folder comes before Dataset. + * Create alphabetical based two-argument comparator to handle library items (including folders) * If sort_key is not present it is set to ''. * @param {str} sort_key key to sort by * @param {str} sort_order order to sort by (asc, desc) * @return {function} two-argument comparator function */ var generateComparator = (sort_key, sort_order) => (itemA, itemB) => { - if (itemA.get("type") === itemB.get("type")) { - if (!itemA.has(sort_key) && !itemB.has(sort_key)) { - return 0; - } else if (!itemA.has(sort_key)) { - return 1; - } else if (!itemB.has(sort_key)) { - return -1; - } - var comparable_itemA_key; - var comparable_itemB_key; - if (typeof itemA.get(sort_key) === "number") { - comparable_itemA_key = itemA.get(sort_key); - comparable_itemB_key = itemB.get(sort_key); - } - else - { - comparable_itemA_key = itemA.get(sort_key).toLowerCase(); - comparable_itemB_key = itemB.get(sort_key).toLowerCase(); - } - if ( comparable_itemA_key > comparable_itemB_key) { - return sort_order === "asc" ? 1 : -1; - } - if (comparable_itemB_key > comparable_itemA_key) { - return sort_order === "asc" ? -1 : 1; - } - - return 0; // equal - } else { - if (itemA.get("type") === "folder") { - return -1; // folder is always before dataset - } + if (!itemA.has(sort_key) && !itemB.has(sort_key)) { + return 0; + } else if (!itemA.has(sort_key)) { return 1; + } else if (!itemB.has(sort_key)) { + return -1; } + var comparable_itemA_key; + var comparable_itemB_key; + if (typeof itemA.get(sort_key) === "number") { + comparable_itemA_key = itemA.get(sort_key); + comparable_itemB_key = itemB.get(sort_key); + } + else + { + comparable_itemA_key = itemA.get(sort_key).toLowerCase(); + comparable_itemB_key = itemB.get(sort_key).toLowerCase(); + } + + if ( comparable_itemA_key > comparable_itemB_key) { + return sort_order === "asc" ? 1 : -1; + } + if (comparable_itemB_key > comparable_itemA_key) { + return sort_order === "asc" ? -1 : 1; + } + + return 0; // equal }; export default { generateComparator: generateComparator diff --git a/client/galaxy/scripts/mvc/rules/rule-definitions.js b/client/galaxy/scripts/mvc/rules/rule-definitions.js index 23ccb754b8f..29909912306 100644 --- a/client/galaxy/scripts/mvc/rules/rule-definitions.js +++ b/client/galaxy/scripts/mvc/rules/rule-definitions.js @@ -42,7 +42,7 @@ const applyRegex = function(regex, target, data, replacement, groupCount) { return null; } if (!replacement) { - groupCount = groupCount && parseInt(groupCount); + groupCount = groupCount && parseInt(groupCount, 10); if (groupCount) { if (match.length != groupCount + 1) { failedCount++; @@ -108,7 +108,7 @@ const RULES = { } }, save: (component, rule) => { - rule.start = parseInt(component.addColumnRownumStart); + rule.start = parseInt(component.addColumnRownumStart, 10); }, apply: (rule, data, sources, columns) => { let rownum = rule.start; @@ -169,7 +169,7 @@ const RULES = { const ruleValue = rule.value; let newRow; if (ruleValue.indexOf("identifier") == 0) { - const identifierIndex = parseInt(ruleValue.substring("identifier".length)); + const identifierIndex = parseInt(ruleValue.substring("identifier".length), 10); newRow = (row, index) => { const newRow = row.slice(); newRow.push(sources[index]["identifiers"][identifierIndex]); @@ -253,7 +253,7 @@ const RULES = { component.addColumnRegexTarget = rule.target_column; component.addColumnRegexExpression = rule.expression; component.addColumnRegexReplacement = rule.replacement; - component.addColumnRegexGroupCount = rule.group_count; + component.addColumnRegexGroupCount = parseInt(rule.group_count); } let addColumnRegexType = "global"; if (component.addColumnRegexGroupCount) { @@ -347,7 +347,7 @@ const RULES = { }, save: (component, rule) => { rule.target_column = component.addColumnSubstrTarget; - rule.length = parseInt(component.addColumnSubstrLength); + rule.length = parseInt(component.addColumnSubstrLength, 10); rule.substr_type = component.addColumnSubstrType; }, apply: (rule, data, sources, columns) => { @@ -403,7 +403,7 @@ const RULES = { function newRow(row) { const newRow = []; for (const index in row) { - if (targets.indexOf(parseInt(index)) == -1) { + if (targets.indexOf(parseInt(index, 10)) == -1) { newRow.push(row[index]); } } @@ -448,7 +448,7 @@ const RULES = { const target = rule.target_column; const invert = rule.invert; const filterFunction = function(el, index) { - const row = data[parseInt(index)]; + const row = data[parseInt(index, 10)]; return regExp.exec(row[target]) ? !invert : invert; }; sources = sources.filter(filterFunction); @@ -477,13 +477,13 @@ const RULES = { component.addFilterCountWhich = "first"; component.addFilterCountInvert = false; } else { - component.addFilterCountN = parseInt(rule.count); + component.addFilterCountN = parseInt(rule.count, 10); component.addFilterCountWhich = rule.which; component.addFilterCountInvert = rule.inverse; } }, save: (component, rule) => { - rule.count = parseInt(component.addFilterCountN); + rule.count = parseInt(component.addFilterCountN, 10); rule.which = component.addFilterCountWhich; rule.invert = component.addFilterCountInvert; }, @@ -528,7 +528,7 @@ const RULES = { const target = rule.target_column; const invert = rule.invert; const filterFunction = function(el, index) { - const row = data[parseInt(index)]; + const row = data[parseInt(index, 10)]; return row[target].length ? !invert : invert; }; sources = sources.filter(filterFunction); @@ -562,7 +562,7 @@ const RULES = { const invert = rule.invert; const value = rule.value; const filterFunction = function(el, index) { - const row = data[parseInt(index)]; + const row = data[parseInt(index, 10)]; return row[target] == value ? !invert : invert; }; sources = sources.filter(filterFunction); @@ -598,7 +598,7 @@ const RULES = { const compare_type = rule.compare_type; const value = rule.value; const filterFunction = function(el, index) { - const row = data[parseInt(index)]; + const row = data[parseInt(index, 10)]; const targetValue = parseFloat(row[target]); let matches; if (compare_type == "less_than") { @@ -730,7 +730,7 @@ const RULES = { const newRow0 = [], newRow1 = []; for (let index in row) { - index = parseInt(index); + index = parseInt(index, 10); if (targets0.indexOf(index) > -1) { newRow0.push(row[index]); } else if (targets1.indexOf(index) > -1) { diff --git a/client/galaxy/scripts/mvc/tool/tools.js b/client/galaxy/scripts/mvc/tool/tools.js index d1831083fb9..ddda58bc452 100644 --- a/client/galaxy/scripts/mvc/tool/tools.js +++ b/client/galaxy/scripts/mvc/tool/tools.js @@ -347,9 +347,7 @@ var ToolSearch = Backbone.Model.extend({ SEARCH_RESERVED_TERMS_FAVORITES: ["#favs", "#favorites", "#favourites"], defaults: { - search_hint_string: "search tools", min_chars_for_search: 3, - clear_btn_url: "", visible: true, query: "", results: null, @@ -635,7 +633,7 @@ var ToolSectionView = BaseView.extend({ var ToolSearchView = Backbone.View.extend({ tagName: "div", id: "tool-search", - className: "bar", + className: "search-input", events: { click: "focus_and_select", @@ -649,7 +647,6 @@ var ToolSearchView = Backbone.View.extend({ if (!this.model.is_visible()) { this.$el.hide(); } - this.$el.find("[title]").tooltip(); return this; }, @@ -774,10 +771,9 @@ var templates = { // the search bar at the top of the tool panel tool_search: _.template( ` - - - ` + placeholder="search tools" autocomplete="off" type="text" /> + + ` ), // the category level container in the tool panel (e.g. 'Get Data', 'Text Manipulation') diff --git a/client/galaxy/scripts/mvc/workflow/workflow-view.js b/client/galaxy/scripts/mvc/workflow/workflow-view.js index eb5449e4268..f4a29b16cf1 100644 --- a/client/galaxy/scripts/mvc/workflow/workflow-view.js +++ b/client/galaxy/scripts/mvc/workflow/workflow-view.js @@ -47,6 +47,7 @@ function reset_tool_search(initValue) { // Reset search input. tool_menu_frame.find("#search-spinner").hide(); + tool_menu_frame.find("#search-clear-btn").show(); if (initValue) { var search_input = tool_menu_frame.find("#tool-search-query"); search_input.val("search tools"); @@ -155,6 +156,7 @@ export default Backbone.View.extend({ } // Start a new ajax-request in X ms $("#search-spinner").show(); + $("#search-clear-btn").hide(); this.timer = window.setTimeout(() => { $.get( self.urls.tool_search, @@ -209,6 +211,7 @@ export default Backbone.View.extend({ $("#search-no-results").show(); } $("#search-spinner").hide(); + $("#search-clear-btn").show(); }, "json" ); diff --git a/client/galaxy/style/scss/base.scss b/client/galaxy/style/scss/base.scss index 4a0bbe2a8e9..2e2cb4c6ede 100644 --- a/client/galaxy/style/scss/base.scss +++ b/client/galaxy/style/scss/base.scss @@ -735,54 +735,6 @@ button { } } -.search-query { - display: inline-block; - padding: 4px; - font-size: $font-size-base; - line-height: $line-height-base; - color: $text-color; - border: 1px solid $border-default-color; - padding-left: 14px !important; - padding-right: 14px; - margin-bottom: 0; - @include border-radius(14px); - background: $brand-white; -} -.search-query:focus { - border-color: darken(rgba(82, 168, 236, 0.8), 15%); - $shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6); - @include box-shadow($shadow); - outline: 0; - outline: thin dotted \9; /* IE6-8 */ -} - -.search-spinner { - position: absolute; - display: none; - right: 6px; - top: 5px; - font-size: 1.4em; - color: #888; -} - -#tool-search { - position: relative; - @extend .mb-2; - #search-clear-btn { - position: absolute; - right: 6px; - top: 6px; - display: block; - font-size: 1.4em !important; - text-decoration: none; - color: #888; - @extend .fa-icon; - &:before { - content: "\f057"; - } - } -} - // Messages .errormessagelarge, @@ -1477,55 +1429,59 @@ div.permissionContainer { .toolMenuContainer { color: $panel-text-color; a { - color: $panel-text-color; + color: $panel-text-color; } background: $panel-bg-color; min-height: 100%; - @extend .m-3; -} - -div.toolSectionWrapper { - @extend .mb-1; } div.toolSectionTitle { font-weight: 500; font-size: $h4-font-size; -} - -div.toolPanelLabel { - @extend .mt-3; - @extend .mb-2; - font-weight: bold; - color: $gray-600; - text-transform: uppercase; -} - -div.toolTitle { - @extend .m-2; - display: block; - .labels { - float: right; - } -} - -div a.tool-link { - text-decoration: none; - display: block; - span.tool-old-link { - text-decoration: underline; - } &:hover { background: darken($panel-bg-color, 5%); } } -div.toolTitleNoSection { - @extend .pb-1; +div.toolPanelLabel { + @extend .py-1; + @extend .px-3; font-weight: bold; + font-size: $h4-font-size; + color: $gray-600; + text-transform: uppercase; +} + +div.toolSectionTitle, div.toolTitle, div.toolTitleNoSection { + display: block; + .labels { + float: right; + } + + a { + @extend .px-3; + @extend .py-1; + text-decoration: none; + display: block; + &:hover { + background: darken($panel-bg-color, 5%); + } + } + + &.text-muted a { + &:hover { + background: inherit; + } + } +} + +div.toolSectionWrapper { + div.toolTitle a, div.toolPanelLabel { + @extend .pl-4; + font-size: inherit; + } } -// Dataset Display Styles #loading_indicator { position: fixed; right: 10px; diff --git a/client/galaxy/style/scss/history.scss b/client/galaxy/style/scss/history.scss index 8bea041c41e..4e4686de8d8 100644 --- a/client/galaxy/style/scss/history.scss +++ b/client/galaxy/style/scss/history.scss @@ -470,21 +470,6 @@ .btn { @extend .btn-sm; } - .search-control { - display: inline-block; - width: 40%; - - .search-clear, - .search-loading { - margin-top: -22px; - } - } - input.search-query { - font-size: 90%; - height: 21px; - line-height: normal; - padding: 2px 2px 1px 2px; - } .open-more-options { padding: 2px 6px 2px 6px; font-size: 100%; diff --git a/client/galaxy/style/scss/theme/blue.scss b/client/galaxy/style/scss/theme/blue.scss index de520a37669..a6d2e196b6d 100644 --- a/client/galaxy/style/scss/theme/blue.scss +++ b/client/galaxy/style/scss/theme/blue.scss @@ -114,7 +114,9 @@ $layout-border: none; // Borders $border-radius-base: 0.1875rem; $border-radius-large: 0.3125rem; +$border-radius-extralarge: 1rem; $border-default-color: $gray-400; +$border-default: 1px solid $border-default-color; // Buttons $btn-default-color: $text-color; diff --git a/client/galaxy/style/scss/ui/search-input.scss b/client/galaxy/style/scss/ui/search-input.scss index 71b0d9290ca..1b35779a931 100644 --- a/client/galaxy/style/scss/ui/search-input.scss +++ b/client/galaxy/style/scss/ui/search-input.scss @@ -1,26 +1,37 @@ -// search bar, see: scripts/jq-plugins/ui/search-input.js .search-input { - .search-query { - width: 100%; - padding-right: 24px; - } + @extend .mb-2; .search-clear, .search-loading { - // it places the icons on the right of the bar (and puts the lotion on its skin) + @extend .mr-2; position: relative; display: inline-block; - float: right; margin-top: -25px; - margin-right: 4px; - font-size: 1.4em; - line-height: 23px; - color: grey; + color: $gray-500; } - .search-clear:hover { - color: $link-color; + color: $brand-info; cursor: pointer; } + .search-loading { + display: none; + } + .search-query { + @extend .px-3; + @extend .py-1; + width: 100%; + display: inline-block; + font-size: $font-size-base; + line-height: $line-height-base; + color: $text-color; + border: $border-default; + @include border-radius($border-radius-extralarge); + max-width: auto; + background: $brand-white; + } + .search-query:focus { + border-color: $brand-info; + outline: 0; + } } diff --git a/client/webpack.config.js b/client/webpack.config.js index 20a17781fd9..b1833ce6a94 100644 --- a/client/webpack.config.js +++ b/client/webpack.config.js @@ -19,7 +19,7 @@ let buildconfig = { generic: ["polyfills", "bundleEntries", "entry/generic"] }, output: { - path: path.join(__dirname, "../", "static/scripts/bundled"), + path: path.join(__dirname, "../", "lib/galaxy/web/framework/static/scripts/bundled"), publicPath: "/static/scripts/bundled/", filename: "[name].bundled.js", chunkFilename: "[name].chunk.js" diff --git a/config/auth_conf.xml.sample b/config/auth_conf.xml.sample deleted file mode 100644 index 34d30693b72..00000000000 --- a/config/auth_conf.xml.sample +++ /dev/null @@ -1,128 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - localdb - - - true - - - diff --git a/config/auth_conf.xml.sample b/config/auth_conf.xml.sample new file mode 120000 index 00000000000..3ccdb59f68e --- /dev/null +++ b/config/auth_conf.xml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/auth_conf.xml.sample \ No newline at end of file diff --git a/config/build_sites.yml.sample b/config/build_sites.yml.sample deleted file mode 100644 index 24a3bed8488..00000000000 --- a/config/build_sites.yml.sample +++ /dev/null @@ -1,22 +0,0 @@ -# Build sites define the builds (dbkeys) available at sites used by display -# applications and the URL to those sites. - -# The `display` attributes on the `ucsc` and `gbrowse` sites replace the -# `ucsc_display_sites` and `gbrowse_display_sites` options in galaxy.ini. -# Because these are used by "old-style" display applications, their types -# cannot change if you want the old-style display links for these sites to -# work. -- type: ucsc - file: tool-data/shared/ucsc/ucsc_build_sites.txt - display: [main,test,archaea,ucla] -- type: gbrowse - file: tool-data/shared/gbrowse/gbrowse_build_sites.txt - display: [modencode,sgd_yeast,tair,wormbase,wormbase_ws120,wormbase_ws140,wormbase_ws170,wormbase_ws180,wormbase_ws190,wormbase_ws200,wormbase_ws204,wormbase_ws210,wormbase_ws220,wormbase_ws225] -- type: ensembl - file: tool-data/shared/ensembl/ensembl_sites.txt -- type: ensembl_data_url - file: tool-data/shared/ensembl/ensembl_sites_data_URL.txt -- type: igv - file: tool-data/shared/igv/igv_build_sites.txt -- type: rviewer - file: tool-data/shared/rviewer/rviewer_build_sites.txt diff --git a/config/build_sites.yml.sample b/config/build_sites.yml.sample new file mode 120000 index 00000000000..7bcce830feb --- /dev/null +++ b/config/build_sites.yml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/build_sites.yml.sample \ No newline at end of file diff --git a/config/container_resolvers_conf.xml.sample b/config/container_resolvers_conf.xml.sample deleted file mode 100644 index 039f2f5bfe1..00000000000 --- a/config/container_resolvers_conf.xml.sample +++ /dev/null @@ -1,59 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/config/container_resolvers_conf.xml.sample b/config/container_resolvers_conf.xml.sample new file mode 120000 index 00000000000..ba8480ebb15 --- /dev/null +++ b/config/container_resolvers_conf.xml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/container_resolvers_conf.xml.sample \ No newline at end of file diff --git a/config/containers_conf.yml.sample b/config/containers_conf.yml.sample deleted file mode 100644 index 71a874cd6ed..00000000000 --- a/config/containers_conf.yml.sample +++ /dev/null @@ -1,278 +0,0 @@ ---- -# Galaxy container interface configuration file -# -# To configure the location of this file, use the `containers_config_file` setting in galaxy.yml. Additionally, the -# containers interface is only used if `enable_beta_containers_interface` is set in galaxy.yml. - -### -### Container Interfaces -### - -# Define container interfaces beneath the top-level `containers` dictionary. By default, a single `_default_` interface -# of type `docker` is defined, equivalent to: - -#containers: -# _default_: -# type: docker - -# The interface name is arbitrary and allows multiple distinct configurations to be defined. The name can be any string, -# but `_default_` is used if a component that uses the containers interface does not specify an interface. Currently -# only Galaxy Interactive Environments use the containers interface, and do not have a way to specify which interface to -# use, so configure under the `_default_` key for now. - -# Additional options can be specified are specific to the container type: - -#containers: -# _default_: -# type: docker -# host: docker.example.org:2376 -# force_tlsverify: yes - -# Keys in `containers` are used to map container consumers to specific # container consumers. If a mapping is not -# configured, the _default_ configuration will be used. An example of multiple container configurations would be: - -#containers: -# _default_: -# type: docker -# ... additional options ... -# example_swarm: -# type: docker_swarm -# ... additional options ... -# - -### -### Container Types and Supported Options -### - -# Command-line equivalent arguments are in [brackets] (if applicable) - -containers: - - # - # Supported options for all container types - # - - _default_: - - # [`--name` (partial)] Prepend this string to the name of containers created - #name_prefix: galaxy_ - - # - # Supported options for `docker` container type - # - - local_docker: - - type: docker - - # [`-H`/`--host] Daemon socket(s) to connect to - #host: null - - # [`--tlsverify`] Use TLS and verify the remote - #force_tlsverify: no - - # [`--cpus`] Number of CPUs (default 0.000) - #cpus: null - - # [`-m`/`--memory`] Memory limit - #memory: null - - # Default image to run if one is not provided to the run method - #image: null - - # - # Supported options for `docker_swarm` container type - # - - swarm: - - type: docker_swarm - - # All of the `docker` interface type arguments are supported. Additionally: - - # [`-H`/`--host] Daemon socket(s) to connect to. This can be a list, which allows failing over to another - # manager when one is down, e.g.: - # host: - # - tcp://swarm1.example.org:2376 - # - tcp://swarm2.example.org:2376 - #host: null - - # [`--reserve-cpu` and `--limit-cpu`] Reserve the given number of CPUs when containers are run to prevent other - # containers from being scheduled on the same node once all of its CPUs are allocated. Additionally, prevent - # container from using more than the given number of CPUs. - #cpus: null - - # [`--reserve-memory` and `--limit-memory`] Reserve the given amount of memory when containers are run to - # prevent other containers from being scheduled on the same node once all of its memory is allocated. - # Additionally, prevent container from using more than the given amount of memory. - #memory: null - - # If set, only nodes whose names begin with this string will be visible to and managed by the swarm manager. - # Note that regardless of whether this is set, the swarm manager will not attempt to control manager nodes - #node_prefix: null - - # Convert image from name[:tag] form to name@digest form when possible, to avoid dependency on the image - # registry (e.g. Docker Hub or wherever the image was pulled from) at service creation time. For details see: - # https://github.com/docker/docker/issues/31427 - #resolve_image_digest: no - - # If a container run request is received that defines volumes to attach, should those volumes be ignored? Swarm - # mode does not support mounting volumes. - #ignore_volumes: no - - # - # For the following `service_create_*_constraint` options, if set, the swarm manager automatically sets a - # corresponding label on nodes it spawns for services with the given constraints. - # - - # [`--constraint node.labels._galaxy_image=={image}`] Automatically create a constraint based on the requested - # image name when new services are created (the image name is the resolved form if `resolve_image_digest` is - # set). This is useful if your nodes do not all have all of the possible images you'll want to run. Note: It's - # currently only possible for nodes to have a single image constraint defined. If your nodes have multiple - # images available, do not use this option. - #service_create_image_constraint: no - - # [`--constraint node.labels._galaxy_cpus=={cpus}`] Automatically create a constraint based on the requested - # number of CPUs (or 1, if `cpus` is unset). This is useful if you are spawning nodes with a mixture of CPU - # counts and want to prevent (for example) 1-cpu services from being scheduled on 2-cpu nodes when 2-cpu jobs - # are waiting. - #service_create_cpus_constraint: no - - # - # Galaxy can manage a Docker swarm using a built-in daemon and callouts to commands to add/remove nodes from - # your swarm. - # - - # Use the swarm manager to manage services on this swarm. Even if your swarm is static, using the manager is - # recommended as the manager will automatically remove services after they have terminated. - #managed: yes - - # Automatically start the swarm manager when new services are created. Useful if you want to run the swarm - # manager but control it separately from Galaxy. - #manager_autostart: yes - - # Configuration dictionary for the swarm manager - manager_conf: - - # - # In order to use the swarm manager for more than just service cleanup, (e.g. its node spawn and destroy - # features), you'll need to configure the `cpus` option in the interface. Otherwise, there are no limits on - # the number of services that will be assigned to a given node, so the swarm manager has no way to determine - # that more nodes should be spawned. - # - # Each service consumes one "slot", and the number of slots available on a node is its number of CPUs - # divided by the value of `cpus`. - # - - # When the swarm manager daemonizes, it writes a pid file so that only one manager will run at a time. This - # is the path to that pid file. {xdg_data_home} will be templated automatically and defaults to - # ~/.local/share as per the XDG specification - #pid_file: '{xdg_data_home}/galaxy_swarm_manager.pid' - - # Program output will be written to the log - #log_file: '{xdg_data_home}/galaxy_swarm_manager.log' - - # Level of log messages (levels are Python logging module level names (case insensitive)) - #log_level: INFO - - # Log message format - #log_format: %(name)s %(levelname)s %(asctime)s %(message)s - - # List of services' environment variables that should be logged when performing periodic service logging. By - # default, none are logged, but `USER_EMAIL` is useful for GIE containers to log the user who is running the - # container. - #log_environment_variables: - # - USER_EMAIL - - # Command to run to spawn new nodes. This command should join the node to the swarm. It is run once per - # unique set of constraints of waiting services. Can include template variables: - # - # - {cpus}: Number of CPUs needed by the requested service(s) or minimum limits - # - {slots}: Number of "slots" needed by the requested service(s) or minimum limits, where slots are CPU - # fractions determined by use of the "cpus" option in the `docker_swarm` section. - # - {image}: Image requested by service - # - {service_ids}: Comma-separated list of service ids with these constraints currently waiting - # - {service_count}: Number of services with these constraints currently waiting - # - # If this command does not block until the node is joined to the swarm, make sure it at least completes that - # step in `spawn_wait_time` once it returns control. How the `spawn_command` exits controls the swarm - # manager's behavior: - # - # - return code: 0, output: space-separated list of spawned nodes: - # Swarm manager will wait for the nodes to appear in the swarm and manage them. You are responsible for ensuring - # the names returned match the name as it will appear in the output of `docker node ls`. - # - return code: 0, output: empty: - # Refusing to allocate nodes and the swarm manager should not attempt to spawn nodes for the given service(s) - # again. This is useful for controlling the maximum number of nodes that will be spawned. - # - return code: 2, output: anything (it will be logged as a message) - # Refusing/failed to allocate nodes but the swarm manager should attempt to spawn nodes for the given - # service(s) again. - # - #spawn_command: /bin/true - - # Command to run to destroy idle nodes. Can include template variables: - # - # - {nodes}: Space-separated list of node names to destroy - # - # This command should block until at least the point at which any nodes being destroyed no longer appear in - # the swarm. It should return the names of nodes destroyed, separated by spaces. - # - #destroy_command: /bin/true - - # Command to run if either of the above commands failed (e.g. to notify an administrator). Can include - # template variables: - # - # - {failed_command}: Command line of the command that failed - # - #command_failure_command: /bin/true - - # Number of times to retry spawn/destroy commands before considering them to have failed, and seconds to - # wait between retries. - #command_retries: 0 - #command_retry_wait: 10 - - # Amount of time to wait for a spawning node to appear in the swarm before considering it failed - #spawn_wait_time: 30 - - # Stop the swarm manager daemon when there are no services or nodes to manage - #terminate_when_idle: yes - - # - # Limits control when the swarm manager will spawn and terminate nodes. - # - - # Number of services that must be waiting to run before attempting to spawn a node. If using constraints, - # then waiting services are grouped by constraint. - #service_wait_count_limit: 0 - - # Number of seconds a service must be waiting before attempting to spawn a node. - #service_wait_time_limit: 5 - - # Number of seconds a node must be idle before terminating it. - #node_idle_limit: 120 - - # Minimum number of worker slots that should be started and active. - #slots_min_limit: 0 - - # Number of spare (unused) worker slots that should be started and active - #slots_min_spare: 0 - - # - # Each set of limits can also be set on a per-constraint basis under the `limits` list. Each member of - # `limits` is a dictionary dictionary with at least the key `constraints`, whose value is a list of - # constraint strings matching constraint strings that jobs will be created with, either by the use of - # `service_create_*_constraint` or manually setting constraints. Other keys match the `node_*` and `slots_*` - # limits above. Any limits unset will default to the "global" limits as set above, or their defaults. - # - # If you are using constraints, you should at least define them in the limits section, even if using all - # "global" values for the limits. This allows the swarm manager to know what node types to keep around if - # you're using `slots_min_limit`. - # - - # Limits section example: - #limits: - # constraints: - # - node.labels._galaxy_image==bgruening/docker-jupyter-notebook:16.01.1 - # - node.labels._galaxy_cpus==1 - # slots_min_limit: 2 - # slots_min_spare: 1 diff --git a/config/containers_conf.yml.sample b/config/containers_conf.yml.sample new file mode 120000 index 00000000000..8653169c427 --- /dev/null +++ b/config/containers_conf.yml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/containers_conf.yml.sample \ No newline at end of file diff --git a/config/data_manager_conf.xml.sample b/config/data_manager_conf.xml.sample deleted file mode 100644 index 04b4637c390..00000000000 --- a/config/data_manager_conf.xml.sample +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample deleted file mode 100644 index 3613a6ee58c..00000000000 --- a/config/datatypes_conf.xml.sample +++ /dev/null @@ -1,935 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample new file mode 120000 index 00000000000..76fe1cddeb6 --- /dev/null +++ b/config/datatypes_conf.xml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/datatypes_conf.xml.sample \ No newline at end of file diff --git a/config/dependency_resolvers_conf.xml.sample b/config/dependency_resolvers_conf.xml.sample deleted file mode 100644 index f111e535818..00000000000 --- a/config/dependency_resolvers_conf.xml.sample +++ /dev/null @@ -1,53 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - diff --git a/config/dependency_resolvers_conf.xml.sample b/config/dependency_resolvers_conf.xml.sample new file mode 120000 index 00000000000..7a876ca5938 --- /dev/null +++ b/config/dependency_resolvers_conf.xml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/dependency_resolvers_conf.xml.sample \ No newline at end of file diff --git a/config/disposable_email_blacklist.conf.sample b/config/disposable_email_blacklist.conf.sample deleted file mode 100644 index 44759454c9c..00000000000 --- a/config/disposable_email_blacklist.conf.sample +++ /dev/null @@ -1,9 +0,0 @@ -If you want to disable registration for users that are using disposable email address -rename this file to disposable_email_blacklist.conf and fill it with the disposable domains -that you want to have blacklisted. Each on its own line without the '@' character as shown below. -Users using emails from these domains will get an error during the registration. - -mailinator.com -sogetthis.com -spamgourmet.com -trashmail.net \ No newline at end of file diff --git a/config/disposable_email_blacklist.conf.sample b/config/disposable_email_blacklist.conf.sample new file mode 120000 index 00000000000..2f93afd3c22 --- /dev/null +++ b/config/disposable_email_blacklist.conf.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/disposable_email_blacklist.conf.sample \ No newline at end of file diff --git a/config/environment_modules_mapping.yml.sample b/config/environment_modules_mapping.yml.sample deleted file mode 100644 index 8fefec6ad55..00000000000 --- a/config/environment_modules_mapping.yml.sample +++ /dev/null @@ -1,47 +0,0 @@ -# This is an example mapping file for the module (http://modules.sourceforge.net/) Dependency resolver (in YAML format) -# -# The goal of this file is to map tool's requirements to existing modules available on your system -# Of course, if the name of a requirement and the name of a module match perfectly, there is no need to map them together through this mapping file. -# -# This is a sample file so the first thing to do to activate the mapping system is to create a copy of this file called "environment_modules_mapping.yml". -# The module dependency resolver is programmed to search and use this YAML file automatically if it exists in the "config" folder of your Galaxy instance. -# Alternatively, you can also use the "mapping_files" attribute of the resolver in the dependency_resolvers_conf.xml file to specify a custom mapping file -# -# Example 1: -# -# Let's say that one of the wrapper installed on your Galaxy instance has the following requirement: -# -# -# PIPITS -# -# -# But unfortunately, the name of the corresponding module file on your system is "pipits_pipeline/1.5.0" -# -# Then, to make Galaxy load/unload the appropriate module, you just have to add the following lines (without the #) to the "environment_modules_mapping.yml" file: -# -#- from: -# name: PIPITS -# version: 1.5.0 -# to: -# name: pipits_pipeline -# version: 1.5.0.6 -# -# -# Example 2: -# -# The requirements section specify a requirement on the PIPITS tool but do not ask for a specific version of it: -# -# -# PIPITS -# -# -# Although, there is no version required you may want to force the loading of a version that is known to run well on your system. -# -# In that case you can add the following lines to the "environment_modules_mapping.yml" file: -# -#- from: -# name: PIPITS -# unversioned: true -# to: -# name: pipits_pipeline -# version: 1.4.0 diff --git a/config/environment_modules_mapping.yml.sample b/config/environment_modules_mapping.yml.sample new file mode 120000 index 00000000000..65a187aa867 --- /dev/null +++ b/config/environment_modules_mapping.yml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/environment_modules_mapping.yml.sample \ No newline at end of file diff --git a/config/error_report.yml.sample b/config/error_report.yml.sample deleted file mode 100644 index cd4ab91ef56..00000000000 --- a/config/error_report.yml.sample +++ /dev/null @@ -1,83 +0,0 @@ -# Each element in this file corresponds to a destination for an error -# report. - -# If you supply the parameter ``user_submission``, and set it to True, -# this error plugin will only be activated when the user uses the bug -# report submission interface. Otherwise, it will be triggered without -# any user interaction (e.g. historical behaviour of sentry plugin) - -# If your plugin is ``user_submission=True``, then supplying -# ``verbose=True`` will cause the plugin to display a message to the end -# user. E.g. the email plugin simply states "Your error report has been -# sent", or the JSON plugin informs the user "Wrote error report to ..." - -# The plugins below are listed with their default values of -# verbose/user_submission, but those are not necessary to provide. - -# The default Email bug reporter. By default, the standard -# configuration is taken from your galaxy.ini -- type: email - verbose: true - user_submission: true - -# Example JSON bug reporter implementation. This just writes the bug -# report as a JSON file to a specified directory and serves as an -# example for others to extend. -# - type: json -# verbose: true -# user_submission: true -# directory: /tmp/reports/ - -# Submit error reports to sentry. If a sentry_dsn is configured in your -# galaxy.ini, then Galaxy will submit the job error to Sentry. You may supply a -# separate DSN for tool reports by supplying a ``custom_dsn`` parameter. -- type: sentry - user_submission: false - -# InfluxDB error reporting backend. You will need to `pip install -# influxdb` in the galaxy virtualenv yourself. This sends well tagged -# errors InfluxDB allowing you to notice relationships between tool errors and -# other infrastructure issues. -# - type: influxdb -# # All arguments prefixed with `influxdb_` are per https://influxdb-python.readthedocs.io/en/latest/api-documentation.html#influxdbclient -# influxdb_host: 127.0.0.1 -# influxdb_port: 8086 -# influxdb_database: galaxy -# influxdb_timeout: 2 - -# Github error reporting backend. You will need to `pip install pygithub` in -# the galaxy virtualenv. This will create a new issue if none exists, and -# comment on existing, open issues. The issues are labelled based on tool ID / -# version and include all of the information the normal emailed bug reports -# include. If you use a private Github Enterprise deployment, you can set -# github_base_url='https://...' and github_api_url='https://api.....' as shown below. -# The 'github_default_repo_only' flag restores the previous behaviour. When this is set -# to true it will automatically only submit to the default git repository. -# - type: github -# verbose: false -# user_submission: true -# github_oauth_token: 00000000000 -# github_base_url: https://github.com -# github_api_url: https://api.github.com -# github_default_repo_owner: galaxyproject -# github_default_repo_name: galaxy -# github_default_repo_only: true - -# GitLab error reporting backend. You will need to `pip install python-gitlab` -# in the galaxy virtualenv. This will create a new issue if none exists, and -# comment on existing, open issues. The issues are labelled based on tool ID / -# version and include all of the information the normal emailed bug reports -# include. If you use a private GitLab deployment, you can set -# gitlab_base_url='https://...'. It supports creating an issue on the git -# repository of the tool by querying the ToolShed where the tool comes from -# (if applicable). Set verbose to true if you want the message to be displayed -# to the user. The 'gitlab_default_repo_only' flags ensures all errors are -# submitted to the default repository only. -# - type: gitlab -# verbose: false -# user_submission: true -# gitlab_base_url: https://gitlab.com -# gitlab_private_token: 00000000000 -# gitlab_default_repo_owner: galaxyproject -# gitlab_default_repo_name: galaxy -# gitlab_default_repo_only: true \ No newline at end of file diff --git a/config/error_report.yml.sample b/config/error_report.yml.sample new file mode 120000 index 00000000000..b702938128c --- /dev/null +++ b/config/error_report.yml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/error_report.yml.sample \ No newline at end of file diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample deleted file mode 100644 index 5b38d4832d6..00000000000 --- a/config/galaxy.yml.sample +++ /dev/null @@ -1,1774 +0,0 @@ -# Galaxy is configured by default to be usable in a single-user development -# environment. To tune the application for a multi-user production -# environment, see the documentation at: -# -# https://docs.galaxyproject.org/en/master/admin/production.html -# -# Throughout this sample configuration file, except where stated otherwise, -# uncommented values override the default if left unset, whereas commented -# values are set to the default value. Relative paths are relative to the root -# Galaxy directory. -# -# Examples of many of these options are explained in more detail in the Galaxy -# Community Hub. -# -# https://galaxyproject.org/admin/config -# -# Config hackers are encouraged to check there before asking for help. -# -uwsgi: - - # The address and port on which to listen. By default, only listen to - # localhost (galaxy will not be accessible over the network). Use - # ':8080' to listen on all available network interfaces. - http: 127.0.0.1:8080 - - # By default uWSGI allocates a very small buffer (4096 bytes) for the - # headers of each request. If you start receiving "invalid request - # block size" in your logs, it could mean you need a bigger buffer. We - # recommend at least 16384. - buffer-size: 16384 - - # Number of web server (worker) processes to fork after the - # application has loaded. If this is set to greater than 1, thunder- - # lock likely should be enabled below. - processes: 1 - - # Number of threads for each web server process. - threads: 4 - - # Number of threads for serving static content and handling internal - # routing requests. - offload-threads: 2 - - # Mapping to serve style content. - static-map: /static/style=static/style/blue - - # Mapping to serve the remainder of the static content. - static-map: /static=static - - # Mapping to serve the favicon. - static-map: /favicon.ico=static/favicon.ico - - # Enable the master process manager. Disabled by default for maximum - # compatibility with CTRL+C, but should be enabled for use with - # --daemon and/or production deployments. - master: false - - # Path to the application's Python virtual environment. If using Conda - # for Galaxy's framework dependencies (not tools!), do not set this. - virtualenv: .venv - - # Path to the application's Python library. - pythonpath: lib - - # The entry point which returns the web application (e.g. Galaxy, - # Reports, etc.) that you are loading. - module: galaxy.webapps.galaxy.buildapp:uwsgi_app() - - # Mount the web application (e.g. Galaxy, Reports, etc.) at the given - # URL prefix. Cannot be used together with 'module:' above. - #mount: /galaxy=galaxy.webapps.galaxy.buildapp:uwsgi_app() - - # Make uWSGI rewrite PATH_INFO and SCRIPT_NAME according to mount- - # points. Set this to true if a URL prefix is used. - manage-script-name: false - - # It is usually a good idea to set this to ``true`` if processes is - # greater than 1. - thunder-lock: false - - # Cause uWSGI to respect the traditional behavior of dying on SIGTERM - # (its default is to brutally reload workers) - die-on-term: true - - # Cause uWSGI to gracefully reload workers and mules upon receipt of - # SIGINT (its default is to brutally kill workers) - hook-master-start: unix_signal:2 gracefully_kill_them_all - - # Cause uWSGI to gracefully reload workers and mules upon receipt of - # SIGTERM (its default is to brutally kill workers) - hook-master-start: unix_signal:15 gracefully_kill_them_all - - # Feature necessary for proper mule signal handling - py-call-osafterfork: true - - # Ensure application threads will run if `threads` is unset. - enable-threads: true - -galaxy: - - # By default, Galaxy uses a SQLite database at - # 'database/universe.sqlite'. You may use a SQLAlchemy connection - # string to specify an external database instead. This string takes - # many options which are explained in detail in the config file - # documentation. - #database_connection: sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE - - # If the server logs errors about not having enough database pool - # connections, you will want to increase these values, or consider - # running more Galaxy processes. - #database_engine_option_pool_size: 5 - - # If the server logs errors about not having enough database pool - # connections, you will want to increase these values, or consider - # running more Galaxy processes. - #database_engine_option_max_overflow: 10 - - # If using MySQL and the server logs the error "MySQL server has gone - # away", you will want to set this to some positive value (7200 should - # work). - #database_engine_option_pool_recycle: -1 - - # If large database query results are causing memory or response time - # issues in the Galaxy process, leave the result on the server - # instead. This option is only available for PostgreSQL and is highly - # recommended. - #database_engine_option_server_side_cursors: false - - # Log all database transactions, can be useful for debugging and - # performance profiling. Logging is done via Python's 'logging' - # module under the qualname - # 'galaxy.model.orm.logging_connection_proxy' - #database_query_profiling_proxy: false - - # If auto-creating a postgres database on startup - it can be based on - # an existing template database. This will set that. This is probably - # only useful for testing but documentation is included here for - # completeness. - #database_template: '' - - # Slow query logging. Queries slower than the threshold indicated - # below will be logged to debug. A value of '0' is disabled. For - # example, you would set this to .005 to log all queries taking longer - # than 5 milliseconds. - #slow_query_log_threshold: 0 - - # Enables a per request sql debugging option. If this is set to true, - # append ?sql_debug=1 to web request URLs to enable detailed logging - # on the backend of SQL queries generated during that request. This is - # useful for debugging slow endpoints during development. - #enable_per_request_sql_debugging: false - - # By default, Galaxy will use the same database to track user data and - # tool shed install data. There are many situations in which it is - # valuable to separate these - for instance bootstrapping fresh Galaxy - # instances with pretested installs. The following option can be used - # to separate the tool shed install database (all other options listed - # above but prefixed with install_ are also available). - #install_database_connection: sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE - - # Setting the following option to true will cause Galaxy to - # automatically migrate the database forward after updates. This is - # not recommended for production use. - #database_auto_migrate: false - - # Wait for database to become available instead of failing - # immediately. - #database_wait: false - - # Number of attempts before failing if database_wait is enabled. - #database_wait_attempts: 60 - - # Time to sleep between attempts if database_wait is enabled (in - # seconds). - #database_wait_sleep: 1.0 - - # Where dataset files are stored. It must accessible at the same path - # on any cluster nodes that will run Galaxy jobs, unless using Pulsar. - #file_path: database/files - - # Where temporary files are stored. It must accessible at the same - # path on any cluster nodes that will run Galaxy jobs, unless using - # Pulsar. - #new_file_path: database/tmp - - # Tool config files, defines what tools are available in Galaxy. Tools - # can be locally developed or installed from Galaxy tool sheds. - # (config/tool_conf.xml.sample will be used if left unset and - # config/tool_conf.xml does not exist). - #tool_config_file: config/tool_conf.xml,config/shed_tool_conf.xml - - # Enable / disable checking if any tools defined in the above non-shed - # tool_config_files (i.e., tool_conf.xml) have been migrated from the - # Galaxy code distribution to the Tool Shed. This functionality is - # largely untested in modern Galaxy releases and has serious issues - # such as #7273 and the possibility of slowing down Galaxy startup, so - # the default and recommended value is False. - #check_migrate_tools: false - - # Tool config maintained by tool migration scripts. If you use the - # migration scripts to install tools that have been migrated to the - # tool shed upon a new release, they will be added to this tool config - # file. - #migrated_tools_config: config/migrated_tools_conf.xml - - # File that contains the XML section and tool tags from all tool panel - # config files integrated into a single file that defines the tool - # panel layout. This file can be changed by the Galaxy administrator - # to alter the layout of the tool panel. If not present, Galaxy will - # create it. - #integrated_tool_panel_config: integrated_tool_panel.xml - - # Default path to the directory containing the tools defined in - # tool_conf.xml. Other tool config files must include the tool_path as - # an attribute in the tag. - #tool_path: tools - - # Path to the directory in which tool dependencies are placed. This - # is used by the Tool Shed to install dependencies and can also be - # used by administrators to manually install or link to dependencies. - # For details, see: https://galaxyproject.org/admin/config/tool- - # dependencies Set the string to None to explicitly disable tool - # dependency handling. If this option is set to none or an invalid - # path, installing tools with dependencies from the Tool Shed will - # fail. - #tool_dependency_dir: database/dependencies - - # The dependency resolvers config file specifies an ordering and - # options for how Galaxy resolves tool dependencies (requirement tags - # in Tool XML). The default ordering is to the use the Tool Shed for - # tools installed that way, use local Galaxy packages, and then use - # Conda if available. See https://github.com/galaxyproject/galaxy/blob - # /dev/doc/source/admin/dependency_resolvers.rst for more information - # on these options. - #dependency_resolvers_config_file: config/dependency_resolvers_conf.xml - - # conda_prefix is the location on the filesystem where Conda packages - # and environments are installed IMPORTANT: Due to a current - # limitation in conda, the total length of the conda_prefix and the - # job_working_directory path should be less than 50 characters! - #conda_prefix: /_conda - - # Override the Conda executable to use, it will default to the one on - # the PATH (if available) and then to /bin/conda - #conda_exec: null - - # Pass debug flag to conda commands. - #conda_debug: false - - # conda channels to enable by default (https://conda.io/docs/user- - # guide/tasks/manage-channels.html) - #conda_ensure_channels: iuc,conda-forge,bioconda,defaults - - # Use locally-built conda packages. - #conda_use_local: false - - # Set to True to instruct Galaxy to look for and install missing tool - # dependencies before each job runs. - #conda_auto_install: false - - # Set to True to instruct Galaxy to install Conda from the web - # automatically if it cannot find a local copy and conda_exec is not - # configured. - #conda_auto_init: true - - # You must set this to True if conda_prefix and job_working_directory - # are not on the same volume, or some conda dependencies will fail to - # execute at job runtime. Conda will copy packages content instead of - # creating hardlinks or symlinks. This will prevent problems with some - # specific packages (perl, R), at the cost of extra disk space usage - # and extra time spent copying packages. - #conda_copy_dependencies: false - - # Certain dependency resolvers (namely Conda) take a considerable - # amount of time to build an isolated job environment in the - # job_working_directory if the job working directory is on a network - # share. Set the following option to True to cache the dependencies - # in a folder. This option is beta and should only be used if you - # experience long waiting times before a job is actually submitted to - # your cluster. - #use_cached_dependency_manager: false - - # By default the tool_dependency_cache_dir is the _cache directory of - # the tool dependency directory - #tool_dependency_cache_dir: /_cache - - # By default, when using a cached dependency manager, the dependencies - # are cached when installing new tools and when using tools for the - # first time. Set this to False if you prefer dependencies to be - # cached only when installing new tools. - #precache_dependencies: true - - # File containing the Galaxy Tool Sheds that should be made available - # to install from in the admin interface (.sample used if default does - # not exist). - #tool_sheds_config_file: config/tool_sheds_conf.xml - - # Set to True to enable monitoring of tools and tool directories - # listed in any tool config file specified in tool_config_file option. - # If changes are found, tools are automatically reloaded. Watchdog ( - # https://pypi.org/project/watchdog/ ) must be installed and available - # to Galaxy to use this option. Other options include 'auto' which - # will attempt to watch tools if the watchdog library is available but - # won't fail to load Galaxy if it is not and 'polling' which will use - # a less efficient monitoring scheme that may work in wider range of - # scenarios than the watchdog default. - #watch_tools: 'false' - - # Set to True to enable monitoring of dynamic job rules. If changes - # are found, rules are automatically reloaded. Takes the same values - # as the 'watch_tools' option. - #watch_job_rules: 'false' - - # As of 18.09, Galaxy defaults to setting up the object store - # configuration for output datasets during the job queue step in job - # handlers. This should generally provide for more robust job - # submission, more configurability, and a better user experience but - # may in some cases slightly slow down the job handler job setup - # process. On the off chance that an admin would like to or need to - # optimize job handlers at the expense of user experience and web - # handling this option will remain for some time by setting this - # option to true. This behavior however should be considered - # deprecated and this option will likely be removed in future versions - # of Galaxy. For more information see - # https://github.com/galaxyproject/galaxy/issues/6513. - #legacy_eager_objectstore_initialization: false - - # Enable Galaxy to fetch Docker containers registered with quay.io - # generated from tool requirements resolved through conda. These - # containers (when available) have been generated using mulled - - # https://github.com/mulled . These containers are highly beta and - # availability will vary by tool. This option will additionally only - # be used for job destinations with Docker enabled. - #enable_beta_mulled_containers: false - - # Container resolvers configuration (beta). Set up a file describing - # container resolvers to use when discovering containers for Galaxy. - # If this is set to None, the default containers loaded is determined - # by enable_beta_mulled_containers. - #containers_resolvers_config_file: '' - - # involucro is a tool used to build Docker or Singularity containers - # for tools from Conda dependencies referenced in tools as - # `requirement`s. The following path is the location of involucro on - # the Galaxy host. This is ignored if the relevant container resolver - # isn't enabled, and will install on demand unless involucro_auto_init - # is set to False. - #involucro_path: database/dependencies/involucro - - # Install involucro as needed to build Docker or Singularity - # containers for tools. Ignored if relevant container resolver is not - # used. - #involucro_auto_init: true - - # Conda channels to use when building Docker or Singularity containers - # using involucro. - #mulled_channels: conda-forge,bioconda - - # Enable automatic polling of relative tool sheds to see if any - # updates are available for installed repositories. Ideally only one - # Galaxy server process should be able to check for repository - # updates. The setting for hours_between_check should be an integer - # between 1 and 24. - #enable_tool_shed_check: false - - # Enable automatic polling of relative tool sheds to see if any - # updates are available for installed repositories. Ideally only one - # Galaxy server process should be able to check for repository - # updates. The setting for hours_between_check should be an integer - # between 1 and 24. - #hours_between_check: 12 - - # Enable use of an in-memory registry with bi-directional - # relationships between repositories (i.e., in addition to lists of - # dependencies for a repository, keep an in-memory registry of - # dependent items for each repository. - #manage_dependency_relationships: false - - # XML config file that contains data table entries for the - # ToolDataTableManager. This file is manually # maintained by the - # Galaxy administrator (.sample used if default does not exist). - #tool_data_table_config_path: config/tool_data_table_conf.xml - - # XML config file that contains additional data table entries for the - # ToolDataTableManager. This file is automatically generated based on - # the current installed tool shed repositories that contain valid - # tool_data_table_conf.xml.sample files. At the time of installation, - # these entries are automatically added to the following file, which - # is parsed and applied to the ToolDataTableManager at server start - # up. - #shed_tool_data_table_config: config/shed_tool_data_table_conf.xml - - # Directory where data used by tools is located. See the samples in - # that directory and the Galaxy Community Hub for help: - # https://galaxyproject.org/admin/data-integration - #tool_data_path: tool-data - - # Directory where Tool Data Table related files will be placed when - # installed from a ToolShed. Defaults to tool_data_path. - #shed_tool_data_path: tool-data - - # Set to True to enable monitoring of the tool_data and - # shed_tool_data_path directories. If changes in tool data table files - # are found, the tool data tables for that data manager are - # automatically reloaded. Watchdog ( - # https://pypi.org/project/watchdog/ ) must be installed and available - # to Galaxy to use this option. Other options include 'auto' which - # will attempt to use the watchdog library if it is available but - # won't fail to load Galaxy if it is not and 'polling' which will use - # a less efficient monitoring scheme that may work in wider range of - # scenarios than the watchdog default. - #watch_tool_data_dir: 'false' - - # File containing old-style genome builds - #builds_file_path: tool-data/shared/ucsc/builds.txt - - # Directory where chrom len files are kept, currently mainly used by - # trackster - #len_file_path: tool-data/shared/ucsc/chrom - - # Datatypes config file(s), defines what data (file) types are - # available in Galaxy (.sample is used if default does not exist). If - # a datatype appears in multiple files, the last definition is used - # (though the first sniffer is used so limit sniffer definitions to - # one file). - #datatypes_config_file: config/datatypes_conf.xml - - # Enable sniffing of compressed datatypes. This can be - # configured/overridden on a per-datatype basis in the - # datatypes_conf.xml file. With this option set to False the - # compressed datatypes will be unpacked before sniffing. - #sniff_compressed_dynamic_datatypes_default: true - - # Disable the 'Auto-detect' option for file uploads - #datatypes_disable_auto: false - - # Visualizations config directory: where to look for individual - # visualization plugins. The path is relative to the Galaxy root dir. - # To use an absolute path begin the path with '/'. This is a comma - # separated list. Defaults to "config/plugins/visualizations". - #visualization_plugins_directory: config/plugins/visualizations - - # Interactive environment plugins root directory: where to look for - # interactive environment plugins. By default none will be loaded. - # Set to config/plugins/interactive_environments to load Galaxy's - # stock plugins. These will require Docker to be configured and have - # security considerations, so proceed with caution. The path is - # relative to the Galaxy root dir. To use an absolute path begin the - # path with '/'. This is a comma separated list. - #interactive_environment_plugins_directory: null - - # To run interactive environment containers in Docker Swarm mode (on - # an existing swarm), set this option to True and set - # `docker_connect_port` in the IE plugin config (ini) file(s) of any - # IE plugins you have enabled and ensure that you are not using any - # `docker run`-specific options in your plugins' `command_inject` - # options (swarm mode services run using `docker service create`, - # which has a different and more limited set of options). This option - # can be overridden on a per-plugin basis by using the `swarm_mode` - # option in the plugin's ini config file. - #interactive_environment_swarm_mode: false - - # Galaxy can run a "swarm manager" service that will monitor - # utilization of the swarm and provision/deprovision worker nodes as - # necessary. The service has its own configuration file. - #swarm_manager_config_file: config/swarm_manager_conf.yml - - # Interactive tour directory: where to store interactive tour - # definition files. Galaxy ships with several basic interface tours - # enabled, though a different directory with custom tours can be - # specified here. The path is relative to the Galaxy root dir. To use - # an absolute path begin the path with '/'. This is a comma separated - # list. - #tour_config_dir: config/plugins/tours - - # Webhooks directory: where to store webhooks - plugins to extend the - # Galaxy UI. By default none will be loaded. Set to - # config/plugins/webhooks/demo to load Galaxy's demo webhooks. To use - # an absolute path begin the path with '/'. This is a comma separated - # list. Add test/functional/webhooks to this list to include the demo - # webhooks used to test the webhook framework. - #webhooks_dir: config/plugins/webhooks - - # Each job is given a unique empty directory as its current working - # directory. This option defines in what parent directory those - # directories will be created. - #job_working_directory: database/jobs_directory - - # If using a cluster, Galaxy will write job scripts and stdout/stderr - # to this directory. - #cluster_files_directory: database/pbs - - # Mako templates are compiled as needed and cached for reuse, this - # directory is used for the cache - #template_cache_path: database/compiled_templates - - # Set to false to disable various checks Galaxy will do to ensure it - # can run job scripts before attempting to execute or submit them. - #check_job_script_integrity: true - - # Number of checks to execute if check_job_script_integrity is - # enabled. - #check_job_script_integrity_count: 35 - - # Time to sleep between checks if check_job_script_integrity is - # enabled (in seconds). - #check_job_script_integrity_sleep: 0.25 - - # Set the default shell used by non-containerized jobs Galaxy-wide. - # This defaults to bash for all jobs and can be overridden at the - # destination level for heterogeneous clusters. conda job resolution - # requires bash or zsh so if this is switched to /bin/sh for instance - # - conda resolution should be disabled. Containerized jobs always use - # /bin/sh - so more maximum portability tool authors should assume - # generated commands run in sh. - #default_job_shell: /bin/bash - - # Citation related caching. Tool citations information maybe fetched - # from external sources such as https://doi.org/ by Galaxy - the - # following parameters can be used to control the caching used to - # store this information. - #citation_cache_type: file - - # Citation related caching. Tool citations information maybe fetched - # from external sources such as https://doi.org/ by Galaxy - the - # following parameters can be used to control the caching used to - # store this information. - #citation_cache_data_dir: database/citations/data - - # Citation related caching. Tool citations information maybe fetched - # from external sources such as https://doi.org/ by Galaxy - the - # following parameters can be used to control the caching used to - # store this information. - #citation_cache_lock_dir: database/citations/lock - - # Configuration file for the object store If this is set and exists, - # it overrides any other objectstore settings. - #object_store_config_file: config/object_store_conf.xml - - # What Dataset attribute is used to reference files in an ObjectStore - # implementation, default is 'id' but can also be set to 'uuid' for - # more de-centralized usage. - #object_store_store_by: id - - # Galaxy sends mail for various things: subscribing users to the - # mailing list if they request it, password resets, reporting dataset - # errors, and sending activation emails. To do this, it needs to send - # mail through an SMTP server, which you may define here (host:port). - # Galaxy will automatically try STARTTLS but will continue upon - # failure. - #smtp_server: '' - - # If your SMTP server requires a username and password, you can - # provide them here (password in cleartext here, but if your server - # supports STARTTLS it will be sent over the network encrypted). - #smtp_username: '' - - # If your SMTP server requires a username and password, you can - # provide them here (password in cleartext here, but if your server - # supports STARTTLS it will be sent over the network encrypted). - #smtp_password: '' - - # If your SMTP server requires SSL from the beginning of the - # connection - #smtp_ssl: false - - # On the user registration form, users may choose to join a mailing - # list. This is the address used to subscribe to the list. Uncomment - # and leave empty if you want to remove this option from the user - # registration form. - #mailing_join_addr: galaxy-announce-join@bx.psu.edu - - # Datasets in an error state include a link to report the error. - # Those reports will be sent to this address. Error reports are - # disabled if no address is set. Also this email is shown as a - # contact to user in case of Galaxy misconfiguration and other events - # user may encounter. - #error_email_to: '' - - # Email address to use in the 'From' field when sending emails for - # account activations, workflow step notifications and password - # resets. We recommend using string in the following format: Galaxy - # Project If not configured, '' will be used. - #email_from: '' - - # URL of the support resource for the galaxy instance. Used in - # activation emails. - #instance_resource_url: https://galaxyproject.org/ - - # E-mail domains blacklist is used for filtering out users that are - # using disposable email address during the registration. If their - # address domain matches any domain in the blacklist, they are refused - # the registration. - #blacklist_file: config/disposable_email_blacklist.conf - - # Registration warning message is used to discourage people from - # registering multiple accounts. Applies mostly for the main Galaxy - # instance. If no message specified the warning box will not be shown. - #registration_warning_message: Please register only one account - we provide this service free of charge and have limited computational resources. Multi-accounts are tracked and will be subjected to account termination and data deletion. - - # User account activation feature global flag. If set to "False", the - # rest of the Account activation configuration is ignored and user - # activation is disabled (i.e. accounts are active since - # registration). The activation is also not working in case the SMTP - # server is not defined. - #user_activation_on: false - - # Activation grace period (in hours). Activation is not forced (login - # is not disabled) until grace period has passed. Users under grace - # period can't run jobs. Enter 0 to disable grace period. - #activation_grace_period: 3 - - # Shown in warning box to users that were not activated yet. In use - # only if activation_grace_period is set. - #inactivity_box_content: Your account has not been activated yet. Feel free to browse around and see what's available, but you won't be able to upload data or run jobs until you have verified your email address. - - # Password expiration period (in days). Users are required to change - # their password every x days. Users will be redirected to the change - # password screen when they log in after their password expires. Enter - # 0 to disable password expiration. - #password_expiration_period: 0 - - # Galaxy Session Timeout This provides a timeout (in minutes) after - # which a user will have to log back in. A duration of 0 disables this - # feature. - #session_duration: 0 - - # You can enter tracking code here to track visitor's behavior through - # your Google Analytics account. Example: UA-XXXXXXXX-Y - #ga_code: '' - - # Galaxy can display data at various external browsers. These options - # specify which browsers should be available. URLs and builds - # available at these browsers are defined in the specified files. If - # use_remote_user = True, display application servers will be denied - # access to Galaxy and so displaying datasets in these sites will - # fail. display_servers contains a list of hostnames which should be - # allowed to bypass security to display datasets. Please be aware - # that there are security implications if this is allowed. More - # details (including required changes to the proxy server config) are - # available in the Apache proxy documentation on the Galaxy Community - # Hub. The list of servers in this sample config are for the UCSC - # Main, Test and Archaea browsers, but the default if left commented - # is to not allow any display sites to bypass security (you must - # uncomment the line below to allow them). - #display_servers: hgw1.cse.ucsc.edu,hgw2.cse.ucsc.edu,hgw3.cse.ucsc.edu,hgw4.cse.ucsc.edu,hgw5.cse.ucsc.edu,hgw6.cse.ucsc.edu,hgw7.cse.ucsc.edu,hgw8.cse.ucsc.edu,lowepub.cse.ucsc.edu - - # To disable the old-style display applications that are hardcoded - # into datatype classes, set enable_old_display_applications = False. - # This may be desirable due to using the new-style, XML-defined, - # display applications that have been defined for many of the - # datatypes that have the old-style. There is also a potential - # security concern with the old-style applications, where a malicious - # party could provide a link that appears to reference the Galaxy - # server, but contains a redirect to a third-party server, tricking a - # Galaxy user to access said site. - #enable_old_display_applications: true - - # Show a message box under the masthead. - #message_box_visible: false - - # Show a message box under the masthead. - #message_box_content: '' - - # Class of the message box under the masthead. Possible values are: - # 'info' (the default), 'warning', 'error', 'done'. - #message_box_class: info - - # Append "/{brand}" to the "Galaxy" text in the masthead. - #brand: '' - - # Format string used when showing date and time information. The - # string may contain: - the directives used by Python time.strftime() - # function (see - # https://docs.python.org/library/time.html#time.strftime ), - $locale - # (complete format string for the server locale), - $iso8601 (complete - # format string as specified by ISO 8601 international standard). - #pretty_datetime_format: $locale (UTC) - - # Location of the configuration file containing extra user - # preferences. - #user_preferences_extra_conf_path: config/user_preferences_extra_conf.yml - - # Default localization for Galaxy UI. Allowed values are listed at the - # end of client/galaxy/scripts/nls/locale.js. With the default value - # (auto), the locale will be automatically adjusted to the user's - # navigator language. Users can override this settings in their user - # preferences if the localization settings are enabled in - # user_preferences_extra_conf.yml - #default_locale: auto - - # URL (with schema http/https) of the Galaxy instance as accessible - # within your local network - if specified used as a default by pulsar - # file staging and Jupyter Docker container for communicating back - # with Galaxy via the API. If you are attempting to set up GIEs on - # Mac OS X with Docker Desktop for Mac and your Galaxy instance runs - # on port 8080 this should be 'http://host.docker.internal:8080'. For - # more details see https://docs.docker.com/docker-for-mac/networking/ - #galaxy_infrastructure_url: http://localhost:8080 - - # If the above URL cannot be determined ahead of time in dynamic - # environments but the port which should be used to access Galaxy can - # be - this should be set to prevent Galaxy from having to guess. For - # example if Galaxy is sitting behind a proxy with REMOTE_USER enabled - # - infrastructure shouldn't talk to Python processes directly and - # this should be set to 80 or 443, etc... If unset this file will be - # read for a server block defining a port corresponding to the webapp. - #galaxy_infrastructure_web_port: 8080 - - # The URL of the page to display in Galaxy's middle pane when loaded. - # This can be an absolute or relative URL. - #welcome_url: /static/welcome.html - - # The URL linked by the "Galaxy/brand" text. - #logo_url: / - - # The URL linked by the "Galaxy Help" link in the "Help" menu. - #helpsite_url: '' - - # The URL linked by the "Wiki" link in the "Help" menu. - #wiki_url: https://galaxyproject.org/ - - # The URL linked by the "Support" link in the "Help" menu. - #support_url: https://galaxyproject.org/support/ - - # The URL linked by the "How to Cite Galaxy" link in the "Help" menu. - #citation_url: https://galaxyproject.org/citing-galaxy - - # The URL linked by the "Search" link in the "Help" menu. - #search_url: https://galaxyproject.org/search/ - - # The URL linked by the "Mailing Lists" link in the "Help" menu. - #mailing_lists_url: https://galaxyproject.org/mailing-lists - - # The URL linked by the "Videos" link in the "Help" menu. - #screencasts_url: https://vimeo.com/galaxyproject - - # Points to the GenomeSpace UI service which will be used by the - # GenomeSpace importer and exporter tools - #genomespace_ui_url: https://gsui.genomespace.org/jsui/ - - # The URL linked by the "Terms and Conditions" link in the "Help" - # menu, as well as on the user registration and login forms and in the - # activation emails. - #terms_url: '' - - # The URL linked by the "Galaxy Q&A" link in the "Help" menu The - # Galaxy Q&A site is under development; when the site is done, this - # URL will be set and uncommented. - #qa_url: null - - # Serve static content, which must be enabled if you're not serving it - # via a proxy server. These options should be self explanatory and so - # are not documented individually. You can use these paths (or ones - # in the proxy server) to point to your own styles. - #static_enabled: true - - # Serve static content, which must be enabled if you're not serving it - # via a proxy server. These options should be self explanatory and so - # are not documented individually. You can use these paths (or ones - # in the proxy server) to point to your own styles. - #static_cache_time: 360 - - # Serve static content, which must be enabled if you're not serving it - # via a proxy server. These options should be self explanatory and so - # are not documented individually. You can use these paths (or ones - # in the proxy server) to point to your own styles. - #static_dir: static/ - - # Serve static content, which must be enabled if you're not serving it - # via a proxy server. These options should be self explanatory and so - # are not documented individually. You can use these paths (or ones - # in the proxy server) to point to your own styles. - #static_images_dir: static/images - - # Serve static content, which must be enabled if you're not serving it - # via a proxy server. These options should be self explanatory and so - # are not documented individually. You can use these paths (or ones - # in the proxy server) to point to your own styles. - #static_favicon_dir: static/favicon.ico - - # Serve static content, which must be enabled if you're not serving it - # via a proxy server. These options should be self explanatory and so - # are not documented individually. You can use these paths (or ones - # in the proxy server) to point to your own styles. - #static_scripts_dir: static/scripts/ - - # Serve static content, which must be enabled if you're not serving it - # via a proxy server. These options should be self explanatory and so - # are not documented individually. You can use these paths (or ones - # in the proxy server) to point to your own styles. - #static_style_dir: static/june_2007_style/blue - - # Serve static content, which must be enabled if you're not serving it - # via a proxy server. These options should be self explanatory and so - # are not documented individually. You can use these paths (or ones - # in the proxy server) to point to your own styles. - #static_robots_txt: static/robots.txt - - # Incremental Display Options - #display_chunk_size: 65536 - - # For help on configuring the Advanced proxy features, see: - # https://docs.galaxyproject.org/en/master/admin/production.html - # Apache can handle file downloads (Galaxy-to-user) via mod_xsendfile. - # Set this to True to inform Galaxy that mod_xsendfile is enabled - # upstream. - #apache_xsendfile: false - - # The same download handling can be done by nginx using X-Accel- - # Redirect. This should be set to the path defined in the nginx - # config as an internal redirect with access to Galaxy's data files - # (see documentation linked above). - #nginx_x_accel_redirect_base: '' - - # If using compression in the upstream proxy server, use this option - # to disable gzipping of library .tar.gz and .zip archives, since the - # proxy server will do it faster on the fly. - #upstream_gzip: false - - # The following default adds a header to web request responses that - # will cause modern web browsers to not allow Galaxy to be embedded in - # the frames of web applications hosted at other hosts - this can help - # prevent a class of attack called clickjacking - # (https://www.owasp.org/index.php/Clickjacking). If you configure a - # proxy in front of Galaxy - please ensure this header remains intact - # to protect your users. Uncomment and leave empty to not set the `X - # -Frame-Options` header. - #x_frame_options: SAMEORIGIN - - # nginx can also handle file uploads (user-to-Galaxy) via - # nginx_upload_module. Configuration for this is complex and explained - # in detail in the documentation linked above. The upload store is a - # temporary directory in which files uploaded by the upload module - # will be placed. - #nginx_upload_store: '' - - # This value overrides the action set on the file upload form, e.g. - # the web path where the nginx_upload_module has been configured to - # intercept upload requests. - #nginx_upload_path: '' - - # Galaxy can also use nginx_upload_module to receive files staged out - # upon job completion by remote job runners (i.e. Pulsar) that - # initiate staging operations on the remote end. See the Galaxy nginx - # documentation for the corresponding nginx configuration. - #nginx_upload_job_files_store: '' - - # Galaxy can also use nginx_upload_module to receive files staged out - # upon job completion by remote job runners (i.e. Pulsar) that - # initiate staging operations on the remote end. See the Galaxy nginx - # documentation for the corresponding nginx configuration. - #nginx_upload_job_files_path: '' - - # Galaxy can upload user files in chunks without using nginx. Enable - # the chunk uploader by specifying a chunk size larger than 0. The - # chunk size is specified in bytes (default: 100MB). - #chunk_upload_size: 104857600 - - # Have Galaxy manage dynamic proxy component for routing requests to - # other services based on Galaxy's session cookie. It will attempt to - # do this by default though you do need to install node+npm and do an - # npm install from `lib/galaxy/web/proxy/js`. It is generally more - # robust to configure this externally, managing it however Galaxy is - # managed. If True, Galaxy will only launch the proxy if it is - # actually going to be used (e.g. for Jupyter). - #dynamic_proxy_manage: true - - # As of 16.04 Galaxy supports multiple proxy types. The original - # NodeJS implementation, alongside a new Golang single-binary-no- - # dependencies version. Valid values are (node, golang) - #dynamic_proxy: node - - # The NodeJS dynamic proxy can use an SQLite database or a JSON file - # for IPC, set that here. - #dynamic_proxy_session_map: database/session_map.sqlite - - # Set the port and IP for the dynamic proxy to bind to, this must - # match the external configuration if dynamic_proxy_manage is False. - #dynamic_proxy_bind_port: 8800 - - # Set the port and IP for the dynamic proxy to bind to, this must - # match the external configuration if dynamic_proxy_manage is False. - #dynamic_proxy_bind_ip: 0.0.0.0 - - # Enable verbose debugging of Galaxy-managed dynamic proxy. - #dynamic_proxy_debug: false - - # The dynamic proxy is proxied by an external proxy (e.g. apache - # frontend to nodejs to wrap connections in SSL). - #dynamic_proxy_external_proxy: false - - # Additionally, when the dynamic proxy is proxied by an upstream - # server, you'll want to specify a prefixed URL so both Galaxy and the - # proxy reside under the same path that your cookies are under. This - # will result in a url like https://FQDN/galaxy-prefix/gie_proxy for - # proxying - #dynamic_proxy_prefix: gie_proxy - - # This attribute governs the minimum length of time between - # consecutive HTTP/WS requests through the proxy, before the proxy - # considers a container as being inactive and kills it. - #dynamic_proxy_golang_noaccess: 60 - - # In order to kill containers, the golang proxy has to check at some - # interval for possibly dead containers. This is exposed as a - # configurable parameter, but the default value is probably fine. - #dynamic_proxy_golang_clean_interval: 10 - - # The golang proxy needs to know how to talk to your docker daemon. - # Currently TLS is not supported, that will come in an update. - #dynamic_proxy_golang_docker_address: unix:///var/run/docker.sock - - # The golang proxy uses a RESTful HTTP API for communication with - # Galaxy instead of a JSON or SQLite file for IPC. If you do not - # specify this, it will be set randomly for you. You should set this - # if you are managing the proxy manually. - #dynamic_proxy_golang_api_key: '' - - # If True, Galaxy will attempt to configure a simple root logger if a - # "loggers" section does not appear in this configuration file. - #auto_configure_logging: true - - # Verbosity of console log messages. Acceptable values can be found - # here: https://docs.python.org/library/logging.html#logging-levels A - # custom debug level of "TRACE" is available for even more verbosity. - #log_level: DEBUG - - # Controls where and how the server logs messages. If unset, the - # default is to log all messages to standard output at the level - # defined by the `log_level` configuration option. Configuration is - # described in the documentation at: - # https://docs.galaxyproject.org/en/master/admin/config_logging.html - #logging: null - - # Print database operations to the server log (warning, quite - # verbose!). - #database_engine_option_echo: false - - # Print database pool operations to the server log (warning, quite - # verbose!). - #database_engine_option_echo_pool: false - - # Turn on logging of application events and some user events to the - # database. - #log_events: true - - # Turn on logging of user actions to the database. Actions currently - # logged are grid views, tool searches, and use of "recently" used - # tools menu. The log_events and log_actions functionality will - # eventually be merged. - #log_actions: true - - # Fluentd configuration. Various events can be logged to the fluentd - # instance configured below by enabling fluent_log. - #fluent_log: false - - # Fluentd configuration. Various events can be logged to the fluentd - # instance configured below by enabling fluent_log. - #fluent_host: localhost - - # Fluentd configuration. Various events can be logged to the fluentd - # instance configured below by enabling fluent_log. - #fluent_port: 24224 - - # Sanitize all HTML tool output. By default, all tool output served - # as 'text/html' will be sanitized thoroughly. This can be disabled - # if you have special tools that require unaltered output. WARNING: - # disabling this does make the Galaxy instance susceptible to XSS - # attacks initiated by your users. - #sanitize_all_html: true - - # Whitelist sanitization file. Datasets created by tools listed in - # this file are trusted and will not have their HTML sanitized on - # display. This can be manually edited or manipulated through the - # Admin control panel -- see "Manage Display Whitelist" - #sanitize_whitelist_file: config/sanitize_whitelist.txt - - # By default Galaxy will serve non-HTML tool output that may - # potentially contain browser executable JavaScript content as plain - # text. This will for instance cause SVG datasets to not render - # properly and so may be disabled by setting the following option to - # True. - #serve_xss_vulnerable_mimetypes: false - - # Return a Access-Control-Allow-Origin response header that matches - # the Origin header of the request if that Origin hostname matches one - # of the strings or regular expressions listed here. This is a comma - # separated list of hostname strings or regular expressions beginning - # and ending with /. E.g. - # mysite.com,google.com,usegalaxy.org,/^[\w\.]*example\.com/ See: - # https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS - #allowed_origin_hostnames: '' - - # Set the following to True to use Jupyter nbconvert to build HTML - # from Jupyter notebooks in Galaxy histories. This process may allow - # users to execute arbitrary code or serve arbitrary HTML. If - # enabled, Jupyter must be available and on Galaxy's PATH, to do this - # run `pip install jinja2 pygments jupyter` in Galaxy's virtualenv. - #trust_jupyter_notebook_conversion: false - - # Debug enables access to various config options useful for - # development and debugging: use_lint, use_profile, use_printdebug and - # use_interactive. It also causes the files used by PBS/SGE - # (submission script, output, and error) to remain on disk after the - # job is complete. - #debug: false - - # Check for WSGI compliance. - #use_lint: false - - # Run the Python profiler on each request. - #use_profile: false - - # Intercept print statements and show them on the returned page. - #use_printdebug: true - - # Enable live debugging in your browser. This should NEVER be enabled - # on a public site. - #use_interactive: false - - # When stopping Galaxy cleanly, how much time to give various - # monitoring/polling threads to finish before giving up on joining - # them. Set to 0 to disable this and terminate without waiting. Among - # others, these threads include the job handler workers, which are - # responsible for preparing/submitting and collecting/finishing jobs, - # and which can cause job errors if not shut down cleanly. If using - # supervisord, consider also increasing the value of `stopwaitsecs`. - # If using job handler mules, consider also setting the `mule-reload- - # mercy` uWSGI option. See the Galaxy Admin Documentation for more. - #monitor_thread_join_timeout: 30 - - # Write thread status periodically to 'heartbeat.log', (careful, uses - # disk space rapidly!). Useful to determine why your processes may be - # consuming a lot of CPU. - #use_heartbeat: false - - # Control the period (in seconds) between dumps. Use -1 to disable. - # Regardless of this setting, if use_heartbeat is enabled, you can - # send a Galaxy process (unless running with uWSGI) SIGUSR1 (`kill - # -USR1`) to force a dump. - #heartbeat_interval: 20 - - # Heartbeat log filename. Can accept the template variables - # {server_name} and {pid} - #heartbeat_log: heartbeat_{server_name}.log - - # Log to Sentry Sentry is an open source logging and error aggregation - # platform. Setting sentry_dsn will enable the Sentry middleware and - # errors will be sent to the indicated sentry instance. This - # connection string is available in your sentry instance under - # -> Settings -> API Keys. - #sentry_dsn: '' - - # Sentry slow request logging. Requests slower than the threshold - # indicated below will be sent as events to the configured Sentry - # server (above, sentry_dsn). A value of '0' is disabled. For - # example, you would set this to .005 to log all queries taking longer - # than 5 milliseconds. - #sentry_sloreq_threshold: 0.0 - - # Log to statsd Statsd is an external statistics aggregator - # (https://github.com/etsy/statsd) Enabling the following options will - # cause galaxy to log request timing and other statistics to the - # configured statsd instance. The statsd_prefix is useful if you are - # running multiple Galaxy instances and want to segment statistics - # between them within the same aggregator. - #statsd_host: null - - # Log to statsd Statsd is an external statistics aggregator - # (https://github.com/etsy/statsd) Enabling the following options will - # cause galaxy to log request timing and other statistics to the - # configured statsd instance. The statsd_prefix is useful if you are - # running multiple Galaxy instances and want to segment statistics - # between them within the same aggregator. - #statsd_port: 8125 - - # Log to statsd Statsd is an external statistics aggregator - # (https://github.com/etsy/statsd) Enabling the following options will - # cause galaxy to log request timing and other statistics to the - # configured statsd instance. The statsd_prefix is useful if you are - # running multiple Galaxy instances and want to segment statistics - # between them within the same aggregator. - #statsd_prefix: galaxy - - # If you are using telegraf to collect these metrics and then sending - # them to InfluxDB, Galaxy can provide more nicely tagged metrics. - # Instead of sending prefix + dot-separated-path, Galaxy will send - # prefix with a tag path set to the page url - #statsd_influxdb: false - - # Add an option to the library upload form which allows administrators - # to upload a directory of files. - #library_import_dir: '' - - # Add an option to the library upload form which allows authorized - # non-administrators to upload a directory of files. The configured - # directory must contain sub-directories named the same as the non- - # admin user's Galaxy login ( email ). The non-admin user is - # restricted to uploading files or sub-directories of files contained - # in their directory. - #user_library_import_dir: '' - - # If user_library_import_dir is set, this option will auto create a - # library import directory for every user (based on their email) upon - # login. - #user_library_import_dir_auto_creation: false - - # For security reasons, users may not import any files that actually - # lie outside of their `user_library_import_dir` (e.g. using symbolic - # links). A list of directories can be allowed by setting the - # following option (the list is comma-separated). Be aware that *any* - # user with library import permissions can import from anywhere in - # these directories (assuming they are able to create symlinks to - # them). - #user_library_import_symlink_whitelist: '' - - # In conjunction or alternatively, Galaxy can restrict user library - # imports to those files that the user can read (by checking basic - # unix permissions). For this to work, the username has to match the - # username on the filesystem. - #user_library_import_check_permissions: false - - # Allow admins to paste filesystem paths during upload. For libraries - # this adds an option to the admin library upload tool allowing admins - # to paste filesystem paths to files and directories in a box, and - # these paths will be added to a library. For history uploads, this - # allows pasting in paths as URIs. (i.e. prefixed with file://). Set - # to True to enable. Please note the security implication that this - # will give Galaxy Admins access to anything your Galaxy user has - # access to. - #allow_path_paste: false - - # Users may choose to download multiple files from a library in an - # archive. By default, Galaxy allows users to select from a few - # different archive formats if testing shows that Galaxy is able to - # create files using these formats. Specific formats can be disabled - # with this option, separate more than one format with commas. - # Available formats are currently 'zip', 'gz', and 'bz2'. - #disable_library_comptypes: null - - # Some sequencer integration features in beta allow you to - # automatically transfer datasets. This is done using a lightweight - # transfer manager which runs outside of Galaxy (but is spawned by it - # automatically). Galaxy will communicate with this manager over the - # port specified here. - #transfer_manager_port: 8163 - - # Boosts are used to customize this instance's toolbox search. The - # higher the boost, the more importance the scoring algorithm gives to - # the given field. Section refers to the tool group in the tool - # panel. Rest of the fields are tool's attributes. - #tool_name_boost: 9.0 - - # Boosts are used to customize this instance's toolbox search. The - # higher the boost, the more importance the scoring algorithm gives to - # the given field. Section refers to the tool group in the tool - # panel. Rest of the fields are tool's attributes. - #tool_section_boost: 3.0 - - # Boosts are used to customize this instance's toolbox search. The - # higher the boost, the more importance the scoring algorithm gives to - # the given field. Section refers to the tool group in the tool - # panel. Rest of the fields are tool's attributes. - #tool_description_boost: 2.0 - - # Boosts are used to customize this instance's toolbox search. The - # higher the boost, the more importance the scoring algorithm gives to - # the given field. Section refers to the tool group in the tool - # panel. Rest of the fields are tool's attributes. - #tool_label_boost: 1.0 - - # Boosts are used to customize this instance's toolbox search. The - # higher the boost, the more importance the scoring algorithm gives to - # the given field. Section refers to the tool group in the tool - # panel. Rest of the fields are tool's attributes. - #tool_stub_boost: 5.0 - - # Boosts are used to customize this instance's toolbox search. The - # higher the boost, the more importance the scoring algorithm gives to - # the given field. Section refers to the tool group in the tool - # panel. Rest of the fields are tool's attributes. - #tool_help_boost: 0.5 - - # Limits the number of results in toolbox search. Can be used to - # tweak how many results will appear. - #tool_search_limit: 20 - - # Enable/ disable Ngram-search for tools. It makes tool search results - # tolerant for spelling mistakes in the query by dividing the query - # into multiple ngrams and search for each ngram - #tool_enable_ngram_search: false - - # Set minimum size of ngrams - #tool_ngram_minsize: 3 - - # Set maximum size of ngrams - #tool_ngram_maxsize: 4 - - # Set tool test data directory. The test framework sets this value to - # 'test-data,https://github.com/galaxyproject/galaxy-test-data.git' - # which will cause Galaxy to clone down extra test data on the fly for - # certain tools distributed with Galaxy but this is likely not - # appropriate for production systems. Instead one can simply clone - # that repository directly and specify a path here instead of a Git - # HTTP repository. - #tool_test_data_directories: test-data - - # Galaxy encodes various internal values when these values will be - # output in some format (for example, in a URL or cookie). You should - # set a key to be used by the algorithm that encodes and decodes these - # values. It can be any string with a length between 5 and 56 bytes. - # One simple way to generate a value for this is with the shell - # command: python -c 'from __future__ import print_function; import - # time; print(time.time())' | md5sum | cut -f 1 -d ' ' - #id_secret: USING THE DEFAULT IS NOT SECURE! - - # User authentication can be delegated to an upstream proxy server - # (usually Apache). The upstream proxy should set a REMOTE_USER - # header in the request. Enabling remote user disables regular logins. - # For more information, see: https://docs.galaxyproject.org/en/master/ - # admin/special_topics/apache.html - #use_remote_user: false - - # If use_remote_user is enabled and your external authentication - # method just returns bare usernames, set a default mail domain to be - # appended to usernames, to become your Galaxy usernames (email - # addresses). - #remote_user_maildomain: '' - - # If use_remote_user is enabled, the header that the upstream proxy - # provides the remote username in defaults to HTTP_REMOTE_USER (the - # 'HTTP_' is prepended by WSGI). This option allows you to change the - # header. Note, you still need to prepend 'HTTP_' to the header in - # this option, but your proxy server should *not* include 'HTTP_' at - # the beginning of the header name. - #remote_user_header: HTTP_REMOTE_USER - - # If use_remote_user is enabled, anyone who can log in to the Galaxy - # host may impersonate any other user by simply sending the - # appropriate header. Thus a secret shared between the upstream proxy - # server, and Galaxy is required. If anyone other than the Galaxy user - # is using the server, then apache/nginx should pass a value in the - # header 'GX_SECRET' that is identical to the one below. - #remote_user_secret: USING THE DEFAULT IS NOT SECURE! - - # If use_remote_user is enabled, you can set this to a URL that will - # log your users out. - #remote_user_logout_href: '' - - # If your proxy and/or authentication source does not normalize e-mail - # addresses or user names being passed to Galaxy - set the following - # option to True to force these to lower case. - #normalize_remote_user_email: false - - # If an e-mail address is specified here, it will hijack remote user - # mechanics (``use_remote_user``) and have the webapp inject a single - # fixed user. This has the effect of turning Galaxy into a single user - # application with no login or external proxy required. Such - # applications should not be exposed to the world. - #single_user: null - - # Administrative users - set this to a comma-separated list of valid - # Galaxy users (email addresses). These users will have access to the - # Admin section of the server, and will have access to create users, - # groups, roles, libraries, and more. For more information, see: - # https://galaxyproject.org/admin/ - #admin_users: '' - - # Force everyone to log in (disable anonymous access). - #require_login: false - - # Show the site's welcome page (see welcome_url) alongside the login - # page (even if require_login is True) - #show_welcome_with_login: false - - # Allow unregistered users to create new accounts (otherwise, they - # will have to be created by an admin). - #allow_user_creation: true - - # Allow administrators to delete accounts. - #allow_user_deletion: false - - # Allow administrators to log in as other users (useful for debugging) - #allow_user_impersonation: false - - # When using LDAP for authentication, allow administrators to pre- - # populate users using an additional form on 'Create new user' - #show_user_prepopulate_form: false - - # Allow users to remove their datasets from disk immediately - # (otherwise, datasets will be removed after a time period specified - # by an administrator in the cleanup scripts run via cron) - #allow_user_dataset_purge: true - - # By default, users' data will be public, but setting this to True - # will cause it to be private. Does not affect existing users and - # data, only ones created after this option is set. Users may still - # change their default back to public. - #new_user_dataset_access_role_default_private: false - - # Expose user list. Setting this to True will expose the user list to - # authenticated users. This makes sharing datasets in smaller galaxy - # instances much easier as they can type a name/email and have the - # correct user show up. This makes less sense on large public Galaxy - # instances where that data shouldn't be exposed. For semi-public - # Galaxies, it may make sense to expose just the username and not - # email, or vice versa. If enable_beta_gdpr is set to True, then this - # option will be overridden and set to False. - #expose_user_name: false - - # Expose user list. Setting this to True will expose the user list to - # authenticated users. This makes sharing datasets in smaller galaxy - # instances much easier as they can type a name/email and have the - # correct user show up. This makes less sense on large public Galaxy - # instances where that data shouldn't be exposed. For semi-public - # Galaxies, it may make sense to expose just the username and not - # email, or vice versa. If enable_beta_gdpr is set to True, then this - # option will be overridden and set to False. - #expose_user_email: false - - # Whitelist for local network addresses for "Upload from URL" dialog. - # By default, Galaxy will deny access to the local network address - # space, to prevent users making requests to services which the - # administrator did not intend to expose. Previously, you could - # request any network service that Galaxy might have had access to, - # even if the user could not normally access it. It should be a comma - # separated list of IP addresses or IP address/mask, e.g. - # 10.10.10.10,10.0.1.0/24,fd00::/8 - #fetch_url_whitelist: null - - # Enables GDPR Compliance mode. This makes several changes to the way - # Galaxy logs and exposes data externally such as removing emails and - # usernames from logs and bug reports. It also causes the delete user - # admin action to permanently redact their username and password, but - # not to delete data associated with the account as this is not - # currently easily implementable. You are responsible for removing - # personal data from backups. This forces expose_user_email and - # expose_user_name to be false, and forces user_deletion to be true to - # support the right to erasure. Please read the GDPR section under - # the special topics area of the admin documentation. - #enable_beta_gdpr: false - - # Enable the new interface for installing tools from Tool Shed via the - # API. Admin menu will list both if enabled. - #enable_beta_ts_api_install: true - - # Enable the new container interface for Interactive Environments - #enable_beta_containers_interface: false - - # Enable beta workflow modules that should not yet be considered part - # of Galaxy's stable API. - #enable_beta_workflow_modules: false - - # Default format for the export of workflows. Possible values are 'ga' - # or 'format2'. - #default_workflow_export_format: ga - - # Following options only apply to workflows scheduled using the legacy - # workflow run API (running workflows via a POST to /api/workflows). - # Force usage of Galaxy's beta workflow scheduler under certain - # circumstances - this workflow scheduling forces Galaxy to schedule - # workflows in the background so initial submission of the workflows - # is significantly sped up. This does however force the user to - # refresh their history manually to see newly scheduled steps (for - # "normal" workflows - steps are still scheduled far in advance of - # them being queued and scheduling here doesn't refer to actual - # cluster job scheduling). Workflows containing more than the - # specified number of steps will always use the Galaxy's beta workflow - # scheduling. - #force_beta_workflow_scheduled_min_steps: 250 - - # Following options only apply to workflows scheduled using the legacy - # workflow run API (running workflows via a POST to /api/workflows). - # Force usage of Galaxy's beta workflow scheduler under certain - # circumstances - this workflow scheduling forces Galaxy to schedule - # workflows in the background so initial submission of the workflows - # is significantly sped up. This does however force the user to - # refresh their history manually to see newly scheduled steps (for - # "normal" workflows - steps are still scheduled far in advance of - # them being queued and scheduling here doesn't refer to actual - # cluster job scheduling). Workflows containing more than the - # specified number of steps will always use the Galaxy's beta workflow - # scheduling. Switch to using Galaxy's beta workflow scheduling for - # all workflows involving collections. - #force_beta_workflow_scheduled_for_collections: false - - # If multiple job handlers are enabled, allow Galaxy to schedule - # workflow invocations in multiple handlers simultaneously. This is - # discouraged because it results in a less predictable order of - # workflow datasets within in histories. - #parallelize_workflow_scheduling_within_histories: false - - # This is the maximum amount of time a workflow invocation may stay in - # an active scheduling state in seconds. Set to -1 to disable this - # maximum and allow any workflow invocation to schedule indefinitely. - # The default corresponds to 1 month. - #maximum_workflow_invocation_duration: 2678400 - - # Specify a maximum number of jobs that any given workflow scheduling - # iteration can create. Set this to a positive integer to prevent - # large collection jobs in a workflow from preventing other jobs from - # executing. This may also mitigate memory issues associated with - # scheduling workflows at the expense of increased total DB traffic - # because model objects are expunged from the SQL alchemy session - # between workflow invocation scheduling iterations. Set to -1 to - # disable any such maximum (the default). - #maximum_workflow_jobs_per_scheduling_iteration: -1 - - # Force serial scheduling of workflows within the context of a - # particular history - #history_local_serial_workflow_scheduling: false - - # Enables and disables OpenID Connect (OIDC) support. - #enable_oidc: false - - # Sets the path to OIDC configuration file. - #oidc_config_file: config/oidc_config.xml - - # Sets the path to OIDC backends configuration file. - #oidc_backends_config_file: config/oidc_backends_config.xml - - # XML config file that allows the use of different authentication - # providers (e.g. LDAP) instead or in addition to local authentication - # (.sample is used if default does not exist). - #auth_config_file: config/auth_conf.xml - - # Optional list of email addresses of API users who can make calls on - # behalf of other users. - #api_allow_run_as: '' - - # Master key that allows many API admin actions to be used without - # actually having a defined admin user in the database/config. Only - # set this if you need to bootstrap Galaxy, you probably do not want - # to set this on public servers. - #master_api_key: changethis - - # Enable access to post-authentication options via OpenID. - #enable_openid: false - - # If OpenID is enabled, consumer cache directory to use. - #openid_consumer_cache_path: database/openid_consumer_cache - - # Enable tool tags (associating tools with tags). This has its own - # option since its implementation has a few performance implications - # on startup for large servers. - #enable_tool_tags: false - - # Enable a feature when running workflows. When enabled, default - # datasets are selected for "Set at Runtime" inputs from the history - # such that the same input will not be selected twice, unless there - # are more inputs than compatible datasets in the history. When False, - # the most recently added compatible item in the history will be used - # for each "Set at Runtime" input, independent of others in the - # Workflow - #enable_unique_workflow_defaults: false - - # The URL to the myExperiment instance being used (omit scheme but - # include port) - #myexperiment_url: www.myexperiment.org:80 - - # Enable Galaxy's "Upload via FTP" interface. You'll need to install - # and configure an FTP server (we've used ProFTPd since it can use - # Galaxy's database for authentication) and set the following two - # options. This should point to a directory containing subdirectories - # matching users' identifier (defaults to e-mail), where Galaxy will - # look for files. - #ftp_upload_dir: '' - - # This should be the hostname of your FTP server, which will be - # provided to users in the help text. - #ftp_upload_site: '' - - # User attribute to use as subdirectory in calculating default - # ftp_upload_dir pattern. By default this will be email so a user's - # FTP upload directory will be ${ftp_upload_dir}/${user.email}. Can - # set this to other attributes such as id or username though. - #ftp_upload_dir_identifier: email - - # Python string template used to determine an FTP upload directory for - # a particular user. - #ftp_upload_dir_template: ${ftp_upload_dir}/${ftp_upload_dir_identifier - - # This should be set to False to prevent Galaxy from deleting uploaded - # FTP files as it imports them. - #ftp_upload_purge: true - - # Enable enforcement of quotas. Quotas can be set from the Admin - # interface. - #enable_quotas: false - - # This option allows users to see the full path of datasets via the - # "View Details" option in the history. This option also exposes the - # command line to non-administrative users. Administrators can always - # see dataset paths. - #expose_dataset_path: false - - # This option allows users to see the job metrics (except for - # environment variables). - #expose_potentially_sensitive_job_metrics: false - - # Enable the API for sample tracking - #enable_legacy_sample_tracking_api: false - - # Allow non-admin users to view available Data Manager options. - #enable_data_manager_user_view: false - - # File where Data Managers are configured (.sample used if default - # does not exist). - #data_manager_config_file: config/data_manager_conf.xml - - # File where Tool Shed based Data Managers are configured. - #shed_data_manager_config_file: config/shed_data_manager_conf.xml - - # Directory to store Data Manager based tool-data; defaults to - # tool_data_path. - #galaxy_data_manager_data_path: tool-data - - # To increase performance of job execution and the web interface, you - # can separate Galaxy into multiple processes. There are more than - # one way to do this, and they are explained in detail in the - # documentation: - # https://docs.galaxyproject.org/en/master/admin/scaling.html By - # default, Galaxy manages and executes jobs from within a single - # process and notifies itself of new jobs via in-memory queues. Jobs - # are run locally on the system on which Galaxy is started. Advanced - # job running capabilities can be configured through the job - # configuration file. - #job_config_file: config/job_conf.xml - - # Description of job running configuration, can be embedded into - # Galaxy configuration or loaded from an additional file with the - # job_config_file option. - #job_config: null - - # Rather than specifying a dependency_resolvers_config_file, the - # definition of the resolvers to enable can be embedded into Galaxy's - # config with this option. This has no effect if a - # dependency_resolvers_config_file is used. - #dependency_resolvers: null - - # When jobs fail due to job runner problems, Galaxy can be configured - # to retry these or reroute the jobs to new destinations. Very fine - # control of this is available with resubmit declarations in - # job_conf.xml. For simple deployments of Galaxy though, the following - # attribute can define resubmission conditions for all job - # destinations. If any job destination defines even one resubmission - # condition explicitly in job_conf.xml - the condition described by - # this option will not apply to that destination. For instance, the - # condition: 'attempt < 3 and unknown_error and (time_running < 300 or - # time_since_queued < 300)' would retry up to two times jobs that - # didn't fail due to detected memory or walltime limits but did fail - # quickly (either while queueing or running). The commented out - # default below results in no default job resubmission condition, - # failing jobs are just failed outright. - #default_job_resubmission_condition: null - - # This option is deprecated, use the `mem-self` handler assignment - # option in the job configuration instead. - #track_jobs_in_database: true - - # This enables splitting of jobs into tasks, if specified by the - # particular tool config. This is a new feature and not recommended - # for production servers yet. - #use_tasked_jobs: false - - # This enables splitting of jobs into tasks, if specified by the - # particular tool config. This is a new feature and not recommended - # for production servers yet. - #local_task_queue_workers: 2 - - # Enable job recovery (if Galaxy is restarted while cluster jobs are - # running, it can "recover" them when it starts). This is not safe to - # use if you are running more than one Galaxy server using the same - # database. - #enable_job_recovery: true - - # Although it is fairly reliable, setting metadata can occasionally - # fail. In these instances, you can choose to retry setting it - # internally or leave it in a failed state (since retrying internally - # may cause the Galaxy process to be unresponsive). If this option is - # set to False, the user will be given the option to retry externally, - # or set metadata manually (when possible). - #retry_metadata_internally: true - - # Very large metadata values can cause Galaxy crashes. This will - # allow limiting the maximum metadata key size (in bytes used in - # memory, not the end result database value size) Galaxy will attempt - # to save with a dataset. Use 0 to disable this feature. The default - # is 5MB, but as low as 1MB seems to be a reasonable size. - #max_metadata_value_size: 5242880 - - # This option will override tool output paths to write outputs to the - # job working directory (instead of to the file_path) and the job - # manager will move the outputs to their proper place in the dataset - # directory on the Galaxy server after the job completes. This is - # necessary (for example) if jobs run on a cluster and datasets can - # not be created by the user running the jobs (e.g. if the filesystem - # is mounted read-only or the jobs are run by a different user than - # the galaxy user). - #outputs_to_working_directory: false - - # If your network filesystem's caching prevents the Galaxy server from - # seeing the job's stdout and stderr files when it completes, you can - # retry reading these files. The job runner will retry the number of - # times specified below, waiting 1 second between tries. For NFS, you - # may want to try the -noac mount option (Linux) or -actimeo=0 - # (Solaris). - #retry_job_output_collection: 0 - - # In the past Galaxy would preserve its Python environment when - # running jobs ( and still does for internal tools packaged with - # Galaxy). This behavior exposes Galaxy internals to tools and could - # result in problems when activating Python environments for tools - # (such as with Conda packaging). The default legacy_only will - # restrict this behavior to tools identified by the Galaxy team as - # requiring this environment. Set this to "always" to restore the - # previous behavior (and potentially break Conda dependency resolution - # for many tools). Set this to legacy_and_local to preserve the - # environment for legacy tools and locally managed tools (this might - # be useful for instance if you are installing software into Galaxy's - # virtualenv for tool development). - #preserve_python_environment: legacy_only - - # Clean up various bits of jobs left on the filesystem after - # completion. These bits include the job working directory, external - # metadata temporary files, and DRM stdout and stderr files (if using - # a DRM). Possible values are: always, onsuccess, never - #cleanup_job: always - - # When running DRMAA jobs as the Galaxy user - # (https://docs.galaxyproject.org/en/master/admin/cluster.html - # #submitting-jobs-as-the-real-user) this script is used to run the - # job script Galaxy generates for a tool execution. - #drmaa_external_runjob_script: sudo -E scripts/drmaa_external_runner.py --assign_all_groups - - # When running DRMAA jobs as the Galaxy user - # (https://docs.galaxyproject.org/en/master/admin/cluster.html - # #submitting-jobs-as-the-real-user) this script is used to kill such - # jobs by Galaxy (e.g. if the user cancels the job). - #drmaa_external_killjob_script: sudo -E scripts/drmaa_external_killer.py - - # When running DRMAA jobs as the Galaxy user - # (https://docs.galaxyproject.org/en/master/admin/cluster.html - # #submitting-jobs-as-the-real-user) this script is used transfer - # permissions back and forth between the Galaxy user and the user that - # is running the job. - #external_chown_script: sudo -E scripts/external_chown_script.py - - # When running DRMAA jobs as the Galaxy user - # (https://docs.galaxyproject.org/en/master/admin/cluster.html - # #submitting-jobs-as-the-real-user) Galaxy can extract the user name - # from the email address (actually the local-part before the @) or the - # username which are both stored in the Galaxy data base. The latter - # option is particularly useful for installations that get the - # authentication from LDAP. Also, Galaxy can accept the name of a - # common system user (eg. galaxy_worker) who can run every job being - # submitted. This user should not be the same user running the galaxy - # system. Possible values are user_email (default), username or - # - #real_system_username: user_email - - # File to source to set up the environment when running jobs. By - # default, the environment in which the Galaxy server starts is used - # when running jobs locally, and the environment set up per the DRM's - # submission method and policy is used when running jobs on a cluster - # (try testing with `qsub` on the command line). - # environment_setup_file can be set to the path of a file on the - # cluster that should be sourced by the user to set up the environment - # prior to running tools. This can be especially useful for running - # jobs as the actual user, to remove the need to configure each user's - # environment individually. - #environment_setup_file: '' - - # Optional file containing job resource data entry fields definition. - # These fields will be presented to users in the tool forms and allow - # them to overwrite default job resources such as number of - # processors, memory and walltime. - #job_resource_params_file: config/job_resource_params_conf.xml - - # Similar to the above parameter, workflows can describe parameters - # used to influence scheduling of jobs within the workflow. This - # requires both a description of the fields available (which defaults - # to the definitions in job_resource_params_file if not set). - #workflow_resource_params_file: config/workflow_resource_params_conf.xml - - # This parameter describes how to map users and workflows to a set of - # workflow resource parameter to present (typically input IDs from - # workflow_resource_params_file). If this this is a function reference - # it will be passed various inputs (workflow model object and user) - # and it should produce a list of input IDs. If it is a path it is - # expected to an XML or YAML file describing how to map group names to - # parameter descriptions (additional types of mappings via these files - # could be implemented but haven't yet - for instance using workflow - # tags to do the mapping). - #workflow_resource_params_mapper: config/workflow_resource_mapper_conf.yml - - # If using job concurrency limits (configured in job_config_file), - # several extra database queries must be performed to determine the - # number of jobs a user has dispatched to a given destination. By - # default, these queries will happen for every job that is waiting to - # run, but if cache_user_job_count is set to True, it will only happen - # once per iteration of the handler queue. Although better for - # performance due to reduced queries, the trade-off is a greater - # possibility that jobs will be dispatched past the configured limits - # if running many handlers. - #cache_user_job_count: false - - # Define toolbox filters (https://galaxyproject.org/user-defined- - # toolbox-filters/) that admins may use to restrict the tools to - # display. - #tool_filters: null - - # Define toolbox filters (https://galaxyproject.org/user-defined- - # toolbox-filters/) that admins may use to restrict the tool labels to - # display. - #tool_label_filters: null - - # Define toolbox filters (https://galaxyproject.org/user-defined- - # toolbox-filters/) that admins may use to restrict the tool sections - # to display. - #tool_section_filters: null - - # Define toolbox filters (https://galaxyproject.org/user-defined- - # toolbox-filters/) that users may use to restrict the tools to - # display. - #user_tool_filters: examples:restrict_upload_to_admins, examples:restrict_encode - - # Define toolbox filters (https://galaxyproject.org/user-defined- - # toolbox-filters/) that users may use to restrict the tool sections - # to display. - #user_tool_section_filters: examples:restrict_text - - # Define toolbox filters (https://galaxyproject.org/user-defined- - # toolbox-filters/) that users may use to restrict the tool labels to - # display. - #user_tool_label_filters: examples:restrict_upload_to_admins, examples:restrict_encode - - # The base module(s) that are searched for modules for toolbox - # filtering (https://galaxyproject.org/user-defined-toolbox-filters/) - # functions. - #toolbox_filter_base_modules: galaxy.tools.toolbox.filters,galaxy.tools.filters - - # Galaxy uses AMQP internally for communicating between processes. - # For example, when reloading the toolbox or locking job execution, - # the process that handled that particular request will tell all - # others to also reload, lock jobs, etc. For connection examples, see - # http://docs.celeryproject.org/projects/kombu/en/latest/userguide/con - # nections.html Without specifying anything here, galaxy will first - # attempt to use your specified database_connection above. If that's - # not specified either, Galaxy will automatically create and use a - # separate sqlite database located in your /database folder - # (indicated in the commented out line below). - #amqp_internal_connection: sqlalchemy+sqlite:///./database/control.sqlite?isolation_level=IMMEDIATE - - # Galaxy real time communication server settings - #enable_communication_server: false - - # Galaxy real time communication server settings - #communication_server_host: http://localhost - - # Galaxy real time communication server settings - #communication_server_port: 7070 - - # persistent_communication_rooms is a comma-separated list of rooms - # that should be always available. - #persistent_communication_rooms: null - - # Allow disabling pbkdf2 hashing of passwords for legacy situations. - # This should normally be left enabled unless there is a specific - # reason to disable it. - #use_pbkdf2: true - diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample new file mode 120000 index 00000000000..221c6850ffd --- /dev/null +++ b/config/galaxy.yml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/galaxy.yml.sample \ No newline at end of file diff --git a/config/job_conf.xml.sample_advanced b/config/job_conf.xml.sample_advanced deleted file mode 100644 index e1e72044eb6..00000000000 --- a/config/job_conf.xml.sample_advanced +++ /dev/null @@ -1,1102 +0,0 @@ - - - - - - - - - ok - 0 - ok - 0 - - - - /sge/lib/libdrmaa.so - - - - - - - galaxy.jobs.rules - - - - - - - - - - - - amqp://guest:guest@localhost:5672// - - http://localhost:8080 - - - - - - - - - - - - - - - - - - - - - - - - - - - - /path/to/kubeconfig - - - - - - - - - - - galaxy_pvc1:/mount_point1,galaxy_pvc2:/mount_point2 - - - - - - - - - - - - - - - - - - - - - - - - - - - - GODOCKER_URL - - USERNAME - - APIKEY - - galaxy - - - - - `chronos_host` - - foo@bar.com - - username - - password - - true - - - - - - - - - - - - - - - - - - - - - - 4 - - True - - - - - - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - - - - - - - - - - - - - - - - - - - busybox:ubuntu-14.04 - /path/to/default/container - - - busybox:ubuntu-14.04 - /path/to/default/container - - - - - - walltime=72:00:00 - - - - True - - True - - - - -Xmx6G - '5' - - - - - - - True - - - "$DRM_SET_VARIABLES_FOR_THIS_JOB" - - - $(mktemp -d /mnt/scratch/fastest/gxyjobXXXXXXXXXXX) - - - - - - - foo - - - - dtd - - - choose_one - - cluster1,cluster2,cluster3 - - - - choose_one - cluster1,cluster2,cluster3 - workflow_invocation,history - - - - burst - local_cluster_8_core,local_cluster_1_core,local_cluster_16_core - shared_cluster_8_core - 50 - - - - - burst - local - burst_if_size - 2 - queued - - - python - to_destination_if_size - - galaxycloudrunner.rules - 1g - galaxycloudrunner - local - - - - python - cloudlaunch_pulsar_burst - galaxycloudrunner.rules - https://launch.usegalaxy.org/cloudlaunch/api/v1 - - 37c46c89bcbea797bc7cd76fee10932d2c6a2389 - - pulsar - - local - - - - - - docker_dispatch - docker_cluster - normal_cluster - - - https://examle.com:8913/ - - 123456789changeme - - - - - - - - - - - - - - - - - - - - /path/to/remote/pulsar/files/staging/ - - - - SecureShell - Torque - foo - foo.example.org - walltime=24:00:00,ncpus=4 - - - - - SecureShell - Slurm - foo - my_host - 2:00:00 - 4 - my_partition - - - - - LocalShell - LSF - 16000 - 8 - BigMem - - - - - 8 - - - - - - - - - - - --time=00:05:00 --nodes=1 - - - - - -l h_rt=96:00:00 - - - --mem-per-cpu=512 - - - - - - - - - --time=00:05:00 --nodes=1 - false - - - - - - - my-docker-registry.org - superbioinfo - my-tool - latest - - - 3 - - - true - - - - true - 1 - 2 - centos:latest - - - - false - - - - true - 512 - 2 - /directory/ - - 2 - - - - - - - - - - - - - memory - processors,memory,time,project - - - - - - - - - - - - - - trackster - - - - - - 2 - - 1 - - 1 - 2 - 1 - - 16 - 100 - - 24:00:00 - - 24:00:00 - - 10GB - - - - - SecureShell - Torque - galaxy - foohost_destination.example.org - walltime=@WALLTIME@,ncpus=@NCPUS@ - - - - diff --git a/config/job_conf.xml.sample_advanced b/config/job_conf.xml.sample_advanced new file mode 120000 index 00000000000..19297cf6cfe --- /dev/null +++ b/config/job_conf.xml.sample_advanced @@ -0,0 +1 @@ +../lib/galaxy/config/sample/job_conf.xml.sample_advanced \ No newline at end of file diff --git a/config/job_conf.xml.sample_basic b/config/job_conf.xml.sample_basic deleted file mode 100644 index 6377386f348..00000000000 --- a/config/job_conf.xml.sample_basic +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - diff --git a/config/job_conf.xml.sample_basic b/config/job_conf.xml.sample_basic new file mode 120000 index 00000000000..9a38a78a624 --- /dev/null +++ b/config/job_conf.xml.sample_basic @@ -0,0 +1 @@ +../lib/galaxy/config/sample/job_conf.xml.sample_basic \ No newline at end of file diff --git a/config/job_metrics_conf.xml.sample b/config/job_metrics_conf.xml.sample deleted file mode 100644 index 1c9c70274e5..00000000000 --- a/config/job_metrics_conf.xml.sample +++ /dev/null @@ -1,138 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/config/job_metrics_conf.xml.sample b/config/job_metrics_conf.xml.sample new file mode 120000 index 00000000000..a5e81247f4c --- /dev/null +++ b/config/job_metrics_conf.xml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/job_metrics_conf.xml.sample \ No newline at end of file diff --git a/config/job_resource_params_conf.xml.sample b/config/job_resource_params_conf.xml.sample deleted file mode 100644 index c9adce5cb97..00000000000 --- a/config/job_resource_params_conf.xml.sample +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/config/job_resource_params_conf.xml.sample b/config/job_resource_params_conf.xml.sample new file mode 120000 index 00000000000..0d48fc27826 --- /dev/null +++ b/config/job_resource_params_conf.xml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/job_resource_params_conf.xml.sample \ No newline at end of file diff --git a/config/lmod_modules_mapping.yml.sample b/config/lmod_modules_mapping.yml.sample deleted file mode 100644 index f916e2ce539..00000000000 --- a/config/lmod_modules_mapping.yml.sample +++ /dev/null @@ -1,47 +0,0 @@ -# This is an example mapping file for the LMOD Dependency resolver (in YAML format) -# -# The goal of this file is to map tool's requirements to existing LMOD modules available on your system -# Of course, if the name of a requirement and the name of a module match perfectly, there is no need to map them together through this mapping file. -# -# This is a sample file so the first thing to do to activate the mapping system is to create a copy of this file called "lmod_modules_mapping.yml". -# The Lmod dependency resolver is programmed to search and use this YAML file automatically if it exists in the "config" folder of your Galaxy instance. -# Alternatively, you can also use the "mapping_files" attribute of the resolver in the dependency_resolvers_conf.xml file to specify a custom mapping file -# -# Example 1: -# -# Let's say that one of the wrapper installed on your Galaxy instance has the following requirement: -# -# -# PIPITS -# -# -# But unfortunately, the name of the corresponding module file on your system is "pipits_pipeline/1.5.0" -# -# Then, to make Galaxy load/unload the appropriate module, you just have to add the following lines (without to the #) to the "lmod_modules_mapping.yml" file: -# -#- from: -# name: PIPITS -# version: 1.5.0 -# to: -# name: pipits_pipeline -# version: 1.5.0.6 -# -# -# Example 2: -# -# The requirements section specify a requirement on the PIPITS tool but do not ask for a specific version of it: -# -# -# PIPITS -# -# -# Although, there is no version required you may want to force the loading of a version that is known to run well on your system. -# -# In that case you can add the following lines to the "lmod_modules_mapping.yml" file: -# -#- from: -# name: PIPITS -# unversioned: true -# to: -# name: pipits_pipeline -# version: 1.4.0 diff --git a/config/lmod_modules_mapping.yml.sample b/config/lmod_modules_mapping.yml.sample new file mode 120000 index 00000000000..6625e71d2b4 --- /dev/null +++ b/config/lmod_modules_mapping.yml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/lmod_modules_mapping.yml.sample \ No newline at end of file diff --git a/config/local_conda_mapping.yml.sample b/config/local_conda_mapping.yml.sample deleted file mode 100644 index bc9191acbb5..00000000000 --- a/config/local_conda_mapping.yml.sample +++ /dev/null @@ -1,2 +0,0 @@ -# See $GALAXY_ROOT/lib/galaxy/tools/deps/resolvers/default_conda_mapping.yml for example mapping - -# additional site-specific mappings can be added to config/local_conda_mapping.yml. diff --git a/config/local_conda_mapping.yml.sample b/config/local_conda_mapping.yml.sample new file mode 120000 index 00000000000..40383fa1381 --- /dev/null +++ b/config/local_conda_mapping.yml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/local_conda_mapping.yml.sample \ No newline at end of file diff --git a/config/migrated_tools_conf.xml.sample b/config/migrated_tools_conf.xml.sample deleted file mode 100644 index 0ed7e75705b..00000000000 --- a/config/migrated_tools_conf.xml.sample +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/config/object_store_conf.xml.sample b/config/object_store_conf.xml.sample deleted file mode 100644 index 12dbf9aef5b..00000000000 --- a/config/object_store_conf.xml.sample +++ /dev/null @@ -1,104 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/config/object_store_conf.xml.sample b/config/object_store_conf.xml.sample new file mode 120000 index 00000000000..71edcdea144 --- /dev/null +++ b/config/object_store_conf.xml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/object_store_conf.xml.sample \ No newline at end of file diff --git a/config/oidc_backends_config.xml.sample b/config/oidc_backends_config.xml.sample deleted file mode 100644 index 90b157b1632..00000000000 --- a/config/oidc_backends_config.xml.sample +++ /dev/null @@ -1,111 +0,0 @@ - - - - - ... - ... - http://localhost:8080/authnz/google/callback - - consent - - - - - - ... - ... - http://localhost:8080/authnz/globus/callback - - consent - - - - ... - ... - ... - http://localhost:8000/authnz/custos/callback - - ... - - - - - - - - - - ... - ... - http://localhost:8080/authnz/elixir/callback - consent - - - diff --git a/config/oidc_backends_config.xml.sample b/config/oidc_backends_config.xml.sample new file mode 120000 index 00000000000..0379005f51f --- /dev/null +++ b/config/oidc_backends_config.xml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/oidc_backends_config.xml.sample \ No newline at end of file diff --git a/config/oidc_config.xml.sample b/config/oidc_config.xml.sample deleted file mode 100644 index f8618dd74dd..00000000000 --- a/config/oidc_config.xml.sample +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - diff --git a/config/oidc_config.xml.sample b/config/oidc_config.xml.sample new file mode 120000 index 00000000000..fabc8f8037a --- /dev/null +++ b/config/oidc_config.xml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/oidc_config.xml.sample \ No newline at end of file diff --git a/config/reports.yml.sample b/config/reports.yml.sample deleted file mode 100644 index 5ca6bdd5008..00000000000 --- a/config/reports.yml.sample +++ /dev/null @@ -1,131 +0,0 @@ -uwsgi: - - # The address and port on which to listen. By default, only listen to - # localhost (reports will not be accessible over the network). Use - # ':9001' to listen on all available network interfaces. - http: 127.0.0.1:9001 - - # By default uWSGI allocates a very small buffer (4096 bytes) for the - # headers of each request. If you start receiving "invalid request - # block size" in your logs, it could mean you need a bigger buffer. We - # recommend at least 16384. - buffer-size: 16384 - - # Number of web server (worker) processes to fork after the - # application has loaded. If this is set to greater than 1, thunder- - # lock likely should be enabled below. - processes: 1 - - # Number of threads for each web server process. - threads: 4 - - # Number of threads for serving static content and handling internal - # routing requests. - offload-threads: 2 - - # Mapping to serve style content. - static-map: /static/style=static/style/blue - - # Mapping to serve the remainder of the static content. - static-map: /static=static - - # Mapping to serve the favicon. - static-map: /favicon.ico=static/favicon.ico - - # Enable the master process manager. Disabled by default for maximum - # compatibility with CTRL+C, but should be enabled for use with - # --daemon and/or production deployments. - master: false - - # Path to the application's Python virtual environment. If using Conda - # for Galaxy's framework dependencies (not tools!), do not set this. - virtualenv: .venv - - # Path to the application's Python library. - pythonpath: lib - - # The entry point which returns the web application (e.g. Galaxy, - # Reports, etc.) that you are loading. - module: galaxy.webapps.reports.buildapp:uwsgi_app() - - # Mount the web application (e.g. Galaxy, Reports, etc.) at the given - # URL prefix. Cannot be used together with 'module:' above. - #mount: /galaxy=galaxy.webapps.galaxy.buildapp:uwsgi_app() - - # Make uWSGI rewrite PATH_INFO and SCRIPT_NAME according to mount- - # points. Set this to true if a URL prefix is used. - manage-script-name: false - - # It is usually a good idea to set this to ``true`` if processes is - # greater than 1. - thunder-lock: false - - # Cause uWSGI to respect the traditional behavior of dying on SIGTERM - # (its default is to brutally reload workers) - die-on-term: true - - # Cause uWSGI to gracefully reload workers and mules upon receipt of - # SIGINT (its default is to brutally kill workers) - hook-master-start: unix_signal:2 gracefully_kill_them_all - - # Cause uWSGI to gracefully reload workers and mules upon receipt of - # SIGTERM (its default is to brutally kill workers) - hook-master-start: unix_signal:15 gracefully_kill_them_all - - # Feature necessary for proper mule signal handling - py-call-osafterfork: true - - # Ensure application threads will run if `threads` is unset. - enable-threads: true - -reports: - - # Verbosity of console log messages. Acceptable values can be found - # here: https://docs.python.org/2/library/logging.html#logging-levels - #log_level: DEBUG - - # Database connection. Galaxy Reports are intended for production - # Galaxy instances, so sqlite (and the default value below) is not - # supported. An SQLAlchemy connection string should be used specify an - # external database. - #database_connection: sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE - - # Where dataset files are stored. - #file_path: database/files - - # Where temporary files are stored. - #new_file_path: database/tmp - - # Mako templates are compiled as needed and cached for reuse, this - # directory is used for the cache - #template_cache_path: database/compiled_templates/reports - - # Configuration for debugging middleware - #debug: false - - # Check for WSGI compliance. - #use_lint: false - - # NEVER enable this on a public site (even test or QA) - #use_interactive: false - - # Write thread status periodically to 'heartbeat.log' (careful, uses - # disk space rapidly!) - #use_heartbeat: true - - # Profiling middleware (cProfile based) - #use_profile: true - - # Mail - #smtp_server: yourserver@yourfacility.edu - - # Mail - #error_email_to: your_bugs@bx.psu.edu - - # Enables GDPR Compliance mode. This makes several changes to the way - # Galaxy logs and exposes data externally such as removing - # emails/usernames from logs and bug reports. You are responsible for - # removing personal data from backups. Please read the GDPR section - # under the special topics area of the admin documentation. - #enable_beta_gdpr: false - diff --git a/config/reports.yml.sample b/config/reports.yml.sample new file mode 120000 index 00000000000..dbc4f3720da --- /dev/null +++ b/config/reports.yml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/reports.yml.sample \ No newline at end of file diff --git a/config/shed_data_manager_conf.xml.sample b/config/shed_data_manager_conf.xml.sample deleted file mode 100644 index 04b4637c390..00000000000 --- a/config/shed_data_manager_conf.xml.sample +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/config/shed_tool_conf.xml.sample b/config/shed_tool_conf.xml.sample deleted file mode 100644 index 0ed7e75705b..00000000000 --- a/config/shed_tool_conf.xml.sample +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/config/shed_tool_data_table_conf.xml.sample b/config/shed_tool_data_table_conf.xml.sample deleted file mode 100644 index fd376e4ed3b..00000000000 --- a/config/shed_tool_data_table_conf.xml.sample +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/config/swarm_manager_conf.yml.sample b/config/swarm_manager_conf.yml.sample deleted file mode 100644 index 31c5632519c..00000000000 --- a/config/swarm_manager_conf.yml.sample +++ /dev/null @@ -1,84 +0,0 @@ ---- -# Galaxy docker swarm manager configuration file -# -# To configure the location of this file, use the `swarm_manager_config_file` -# setting in galaxy.ini - -# When the swarm manager daemonizes, it writes a pid file so that only one -# manager will run at a time. This is the path to that pid file. -# {xdg_data_home} will be templated automatically and defaults to -# ~/.local/share as per the XDG specification -#pid_file: '{xdg_data_home}/galaxy_swarm_manager.pid' - -# Program output will be written to the log -#log_file: '{xdg_data_home}/galaxy_swarm_manager.log' - -# As with GIE plugins, you can modify the base docker command ({docker_args} -# must be present and will be filled in with the docker subcommand and -# arguments) -#command: 'docker {docker_args}' - -# Managed services should be started with this string at the beginning of their -# name. It should match the value of CONTAINER_NAME_PREFIX in -# lib/galaxy/web/base/interactive_environments.py, so you should not change -# this unless you change both. -#service_prefix: galaxy_gie_ - -# Limits: -# -# - max_waiting_services: number of services that should be waiting of each -# "CPU class" (number of CPUs requested e.g. with --reserve-cpu) before -# attempting to spawn a node -# - max_wait_time: number of seconds a service should be waiting before -# attempting to spawn a node -# - max_node_idle_time: number of seconds a node should be idle before -# terminating it -# - max_node_counts: a dictionary controlling the maximum number of nodes of -# each CPU class that the swarm manager will attempt to spawn, e.g.: -# max_node_counts: -# 1: 10 # spawn up to 10 x 1-CPU nodes -# 2: 3 # spawn up to 3 x 2-CPU nodes -# 4: 1 # spawn up to 1 x 4-CPU nodes -#max_waiting_services: 0 -#max_wait_time: 5 -#max_node_idle_time: 120 -#max_node_counts: {} - -# If set, only manage nodes whose swarm hostnames begin with this prefix. -# Otherwise, attempt to manage all nodes -#node_prefix: null - -# Amount of time to wait for a spawning node to appear in `docker node ls` -# before considering it failed -#spawn_wait_time: 30 - -# Command to run to spawn new nodes. This command should join the node to the -# swarm. Can include template variables: -# - {cpu_class}: CPU class as explained above (the value of --reserve-cpu) -# - {cpus_needed}: Total number of CPUs of the given class needed to run the -# waiting services -# If this command does not block until the node is joined to the swarm, make -# sure it at least completes that step in `spawn_wait_time` once it returns -# control. This command should return a space-separated list of nodes. If the -# nodes have a different number of CPUs than the class that they were started -# for, you can include that number after a colon (e.g. `node1:4`). -#spawn_command: /bin/true - -# Command to run to destroy idle nodes. Can include template variables: -# - {nodes}: Space-separated list of node names to destroy -# This command should block until at least the point at which any nodes being -# deallocated no longer appear in `docker node ls`. -#destroy_command: /bin/true - -# Command to run if either of the above commands failed (e.g. to notify an -# administrator). Can include template variables: -# - {failed_command}: Command line of the command that failed -#command_failure_command: /bin/true - -# Number of times to retry spawn/destroy commands before considering them to -# have failed, and seconds to wait between retries -#command_retries: 0 -#command_retry_wait: 10 - -# Stop the swarm manager daemon when there are no services or nodes to manage -#terminate_when_idle: True diff --git a/config/swarm_manager_conf.yml.sample b/config/swarm_manager_conf.yml.sample new file mode 120000 index 00000000000..9118a9aa59f --- /dev/null +++ b/config/swarm_manager_conf.yml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/swarm_manager_conf.yml.sample \ No newline at end of file diff --git a/config/tool_conf.xml.main b/config/tool_conf.xml.main deleted file mode 100644 index 99aa2052798..00000000000 --- a/config/tool_conf.xml.main +++ /dev/null @@ -1,142 +0,0 @@ - - -
- - - - - - - - - - - - - - - - - -
-
- - -
-
- -
-
- - - - - - - - - -
-
- -
-
- - - - - - - - -
-
- - - -
-
- - - -
-
- - - - - - - - - - - - -
-
- - - -
-
- - -
-
-
-
- - - - - - - - - - - -
-
-
-
-
-
diff --git a/config/tool_conf.xml.main b/config/tool_conf.xml.main new file mode 120000 index 00000000000..221f0e6b05c --- /dev/null +++ b/config/tool_conf.xml.main @@ -0,0 +1 @@ +../lib/galaxy/config/sample/tool_conf.xml.main \ No newline at end of file diff --git a/config/tool_conf.xml.sample b/config/tool_conf.xml.sample deleted file mode 100644 index aeb42d29f3b..00000000000 --- a/config/tool_conf.xml.sample +++ /dev/null @@ -1,129 +0,0 @@ - - -
- - - - - - - - - - - - - - - - - - - - - -
-
- - -
-
- - - - - - - - - - - - - -
-
- -
-
- -
-
- - - - - - - - - - - - - - - - -
-
- - - - - - - - -
-
- - - -
-
- - - -
-
- - - - - - - - - - - - -
-
- - - -
-
- - -
-
- -
-
- - - - - - - -
-
diff --git a/config/tool_conf.xml.sample b/config/tool_conf.xml.sample new file mode 120000 index 00000000000..021c0404b98 --- /dev/null +++ b/config/tool_conf.xml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/tool_conf.xml.sample \ No newline at end of file diff --git a/config/tool_data_table_conf.xml.sample b/config/tool_data_table_conf.xml.sample deleted file mode 100644 index 8fe7b04a5f7..00000000000 --- a/config/tool_data_table_conf.xml.sample +++ /dev/null @@ -1,108 +0,0 @@ - - - - - value, dbkey, name, path - -
- - - value, dbkey, formats, name, path - -
- - - value, name, path - -
- - - value, name, path - -
- - - value, name, path - -
- - - value, dbkey, name, path - -
- - - value, dbkey, name, path - -
- - - name, value, dbkey, species - -
- - - value, dbkey, name, path - -
- - - value, name, path - -
- - - value, name, path - -
- - - value, dbkey, name, path - -
- - - value, dbkey, name, path - -
- - - value, dbkey, name, path - -
- - - value, path - -
- - - name, url, value - -
- - - dbkey, name, value - -
- - - value, name, url - -
- - - value, name, url - -
- - - value, name, url - -
- - - value, name, url - -
-
diff --git a/config/tool_data_table_conf.xml.sample b/config/tool_data_table_conf.xml.sample new file mode 120000 index 00000000000..ae59f07f32b --- /dev/null +++ b/config/tool_data_table_conf.xml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/tool_data_table_conf.xml.sample \ No newline at end of file diff --git a/config/tool_destinations.yml.sample b/config/tool_destinations.yml.sample deleted file mode 100644 index 9f96b3e8d6f..00000000000 --- a/config/tool_destinations.yml.sample +++ /dev/null @@ -1,313 +0,0 @@ -# Dynamic Tool Destinations config has four main levels (or key properties) - tools, -# default_destination, users, and verbose. -# -# Under tools is where you put the IDs of the tools. Tool IDs must be -# distinct. In each tool is a list of rules, each of which contain rule-specific -# parameters. Dynamic Tool Destination is set up to allow and fix a few light errors -# in the config, but it's best to follow the specified template. -# -# default_destination, the second key property, specifies which global destination -# to default to in case none of the rules apply. -# -# users, the third key property, can be used to assign a priority to a user. Three priorities -# are available low, med, and high. By default all users are assumed to be med priority. If -# a rule has priority destinations the user will be set to the destination which matches their priority -# or if their priority isn't available they will be matched to the next closest one. -# -# A fourth key property is verbose. When this is set to True, Dynamic Tool Destination -# gives much more descriptive output regarding the steps it's taking in mapping your tools to -# the appropriate destinations, including config validation and any potential errors found in the -# config. -# -# Note: Configurations can be validated with the following command: -# -# python lib/galaxy/jobs/dynamic_tool_destination.py -c -# -# or -# -# python lib/galaxy/jobs/dynamic_tool_destination.py -c /path/to/tool_destinations.yml -# -# The general template is as follows (note that this template -# does not use quote symbols), using spades and smalt as an example -# (spades for showing what each field is for, and smalt -# to give a fairly real-world example): -# -# Ex: -# -# tools: -# spades: -# rules: -# - rule_type: what kind of rule is it? -# nice_value: what kind of priority does this rule have over others? -# destination: how should this tool be run? -# lower_bound: what's the max file size? -# upper_bound: what's the minimum file size? -# users: #(optional) which users does this rule apply to -# - user@example.com -# default_destination: this tool-specific field is optional -# smalt_map: -# rules: -# - rule_type: file_size -# nice_value: 0 -# lower_bound: 0 -# upper_bound: 2 GB -# destination: cluster_low_4 -# - rule_type: file_size -# nice_value: 0 -# lower_bound: 2 GB -# upper_bound: 4 GB -# destination: cluster_low_8 -# - rule_type: file_size -# nice_value: 0 -# lower_bound: 4 GB -# upper_bound: Infinity -# destination: cluster_low_16 -# default_destination: cluster_default -# users: -# user_low@example.com: -# priority: low -# user_med@example.com: -# priority: med -# user_high@example.com: -# priority: high -# default_destination: this global field is mandatory -# verbose: True -# -# -# -# Looking at this example, some things must be clarified: each entry in the list of -# rules per tool is specified by '-'. Per rule, regardless of rule type, -# the following fields are mandatory: -# rule_type, nice_value, and destination. -# -# Some of the other fields are mandatory only for specific rule types, which will be -# further discussed below. -# -# Starting with rule_type, there are currently 3 rule types: file_size, records, -# and arguments. -# -# file_size and records rules are based on how large the files are: if they fall -# within specified limits, then the rule is satisfied, and the tool may proceed -# with the appropriate destination. -# -# file_size and records rules have the following required parameters on top of the base -# mandatory parameters: -# upper_bound -# lower_bound -# -# Bounds are allowed to be specified in bytes (48000 for example) or a higher size unit, -# including the unit abbreviation (4 GB or 10 TB for example). Additionally, upper_bound -# is allowed to be Infinite; simply specify Infinite in order to do so. -# -# **The rule will allow the lower_bound, up to but not including the upper_bound -# -# The third rule_type is arguments, which has arguments as a mandatory parameter on top of -# the base mandatory parameters. The arguments parameter is specified using the following -# template: -# -# -# arguments: -# argument_name: the_argument -# -# -# A real world example is shown below: -# -# -# tools: -# spades: -# rules: -# - rule_type: arguments -# nice_value: -19 -# destination: fail -# fail_message: Don't do that -# arguments: -# my_section: -# my_condition: -# careful: true -# default_destination: cluster_low -# default_destination: cluster -# verbose: False -# -# -# Next up, nice_value is used for prioritizing rules over others in case two rules -# match. nice_value basically translates to, "the higher the nice_value, the 'nicer' -# the tool is about being picked last". So based off of that idea, a rule with a nice -# value of -5 is guaranteed to be picked over a rule with a nice value of 10. nice_value -# is allowed to go from -20 to 20. If two rules have the same nice value and both were -# satisfied, the first rule in the config file will be picked. In summary, first-come- -# first-serve basis unless nice_value overrides that. -# -# -# Finally, destination simply refers to the specific way the tool will run. Each -# destination ID refers to a specific configuration to run the tool with. -# -# Some rules may call for the job to fail if certain conditions are encountered. In -# this case, destination simply refers to 'fail'. -# -# For example, the following rule is set to fail the job if a file that is too large -# (more than 4GB) is encountered: -# -# -# tools: -# spades: -# rules: -# - rule_type: file_size -# nice_value: 0 -# destination: fail -# fail_message: Data too large -# lower_bound: 4 GB -# upper_bound: Infinity -# -# -# As shown above, a rule with 'fail' as the destination requires an additional -# parameter, 'fail_message', which DynamicToolDestination uses to print a helpful error -# message to the user indicating why the job failed (showing up inside the job log in -# Galaxy's history panel). - - -tools: - spades: - rules: - - rule_type: file_size - nice_value: 0 - lower_bound: 0 - upper_bound: 10 MB - fail_message: Too few reads for spades to work - destination: fail - - rule_type: file_size - nice_value: 0 - lower_bound: 10 MB - upper_bound: 2 GB - destination: - priority: - low: slurm_low_24 - med: slurm_med_24 - high: slurm_high_24 - - rule_type: file_size - nice_value: 0 - lower_bound: 2 GB - upper_bound: 4 GB - destination: - priority: - low: slurm_low_48 - med: slurm_med_48 - high: slurm_high_48 - - rule_type: file_size - nice_value: 0 - lower_bound: 4 GB - upper_bound: Infinity - fail_message: Too much data, shoudn't run - destination: fail - - rule_type: arguments - nice_value: 0 - arguments: - careful: true - destination: - priority: - low: slurm_low_48 - med: slurm_med_48 - high: slurm_high_48 - default_destination: - priority: - low: slurm_low_16 - med: slurm_med_16 - high: slurm_high_16 - - srst2: - rules: - - rule_type: records - nice_value: 0 - lower_bound: 0 - upper_bound: 100 - destination: - priority: - low: slurm_low_4 - med: slurm_med_4 - high: slurm_high_4 - - rule_type: records - nice_value: 0 - lower_bound: 100 - upper_bound: 1000 - destination: - priority: - low: slurm_low_8 - med: slurm_med_8 - high: slurm_high_8 - - rule_type: records - nice_value: 0 - lower_bound: 1000 - upper_bound: 10000 - destination: - priority: - low: slurm_low_8 - med: slurm_med_8 - high: slurm_high_8 - - rule_type: records - nice_value: 0 - lower_bound: 1000000 - upper_bound: Infinity - fail_message: Using the wrong tool - destination: fail - - smalt_map: - rules: - - rule_type: file_size - nice_value: 0 - lower_bound: 0 - upper_bound: 2 GB - destination: - priority: - low: slurm_low_4 - med: slurm_med_4 - high: slurm_high_4 - - rule_type: file_size - nice_value: 0 - lower_bound: 2 GB - upper_bound: 4 GB - destination: - priority: - low: slurm_low_8 - med: slurm_med_8 - high: slurm_high_8 - - rule_type: file_size - nice_value: 0 - lower_bound: 4 GB - upper_bound: Infinity - destination: - priority: - low: slurm_low_16 - med: slurm_med_16 - high: slurm_high_16 - - fastqc: - rules: - - rule_type: file_size - nice_value: 0 - lower_bound: 0 - upper_bound: 400 MB - fail_message: File size too small - destination: fail - - rule_type: file_size - nice_value: 0 - lower_bound: 400 MB - upper_bound: Infinity - destination: - priority: - low: slurm_low_4 - med: slurm_med_4 - high: slurm_high_4 - default_destination: - priority: - low: slurm_low_16 - med: slurm_med_16 - high: slurm_high_16 - -default_destination: - priority: - low: slurm_low_default - med: slurm_med_default - high: slurm_high_default -users: - user@email.com: - priority: high -verbose: True diff --git a/config/tool_destinations.yml.sample b/config/tool_destinations.yml.sample new file mode 120000 index 00000000000..31315829644 --- /dev/null +++ b/config/tool_destinations.yml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/tool_destinations.yml.sample \ No newline at end of file diff --git a/config/tool_shed.yml.sample b/config/tool_shed.yml.sample deleted file mode 100644 index 59495ea0cd0..00000000000 --- a/config/tool_shed.yml.sample +++ /dev/null @@ -1,385 +0,0 @@ -uwsgi: - - # The address and port on which to listen. By default, only listen to - # localhost (tool_shed will not be accessible over the network). Use - # ':9009' to listen on all available network interfaces. - http: 127.0.0.1:9009 - - # By default uWSGI allocates a very small buffer (4096 bytes) for the - # headers of each request. If you start receiving "invalid request - # block size" in your logs, it could mean you need a bigger buffer. We - # recommend at least 16384. - buffer-size: 16384 - - # Number of web server (worker) processes to fork after the - # application has loaded. If this is set to greater than 1, thunder- - # lock likely should be enabled below. - processes: 1 - - # Number of threads for each web server process. - threads: 4 - - # Number of threads for serving static content and handling internal - # routing requests. - offload-threads: 2 - - # Mapping to serve style content. - static-map: /static/style=static/style/blue - - # Mapping to serve the remainder of the static content. - static-map: /static=static - - # Mapping to serve the favicon. - static-map: /favicon.ico=static/favicon.ico - - # Enable the master process manager. Disabled by default for maximum - # compatibility with CTRL+C, but should be enabled for use with - # --daemon and/or production deployments. - master: false - - # Path to the application's Python virtual environment. If using Conda - # for Galaxy's framework dependencies (not tools!), do not set this. - virtualenv: .venv - - # Path to the application's Python library. - pythonpath: lib - - # The entry point which returns the web application (e.g. Galaxy, - # Reports, etc.) that you are loading. - module: galaxy.webapps.tool_shed.buildapp:uwsgi_app() - - # Mount the web application (e.g. Galaxy, Reports, etc.) at the given - # URL prefix. Cannot be used together with 'module:' above. - #mount: /galaxy=galaxy.webapps.galaxy.buildapp:uwsgi_app() - - # Make uWSGI rewrite PATH_INFO and SCRIPT_NAME according to mount- - # points. Set this to true if a URL prefix is used. - manage-script-name: false - - # It is usually a good idea to set this to ``true`` if processes is - # greater than 1. - thunder-lock: false - - # Cause uWSGI to respect the traditional behavior of dying on SIGTERM - # (its default is to brutally reload workers) - die-on-term: true - - # Cause uWSGI to gracefully reload workers and mules upon receipt of - # SIGINT (its default is to brutally kill workers) - hook-master-start: unix_signal:2 gracefully_kill_them_all - - # Cause uWSGI to gracefully reload workers and mules upon receipt of - # SIGTERM (its default is to brutally kill workers) - hook-master-start: unix_signal:15 gracefully_kill_them_all - - # Feature necessary for proper mule signal handling - py-call-osafterfork: true - - # Ensure application threads will run if `threads` is unset. - enable-threads: true - -tool_shed: - - # Verbosity of console log messages. Acceptable values can be found - # here: https://docs.python.org/library/logging.html#logging-levels - #log_level: DEBUG - - # By default, the Tool Shed uses a SQLite database at - # 'database/community.sqlite'. You may use a SQLAlchemy connection - # string to specify an external database instead. This string takes - # many options which are explained in detail in the config file - # documentation. - #database_connection: sqlite:///./database/community.sqlite?isolation_level=IMMEDIATE - - # Where the hgweb.config file is stored. The default is the Galaxy - # installation directory. - #hgweb_config_dir: null - - # Where Tool Shed repositories are stored. - #file_path: database/community_files - - # Where temporary files are stored. - #new_file_path: database/tmp - - # File containing old-style genome builds - #builds_file_path: tool-data/shared/ucsc/builds.txt - - # Format string used when showing date and time information. The - # string may contain: - the directives used by Python time.strftime() - # function (see - # https://docs.python.org/library/time.html#time.strftime ), - $locale - # (complete format string for the server locale), - $iso8601 (complete - # format string as specified by ISO 8601 international standard). - #pretty_datetime_format: $locale (UTC) - - # -- Repository and Tool search Using the script located at - # scripts/build_ts_whoosh_index.py you can generate search index and - # allow full text API searching over the repositories and tools within - # the Tool Shed given that you specify the following two config - # options. - #toolshed_search_on: true - - # -- Repository and Tool search Using the script located at - # scripts/build_ts_whoosh_index.py you can generate search index and - # allow full text API searching over the repositories and tools within - # the Tool Shed given that you specify the following two config - # options. - #whoosh_index_dir: database/toolshed_whoosh_indexes - - # For searching repositories at /api/repositories: - #repo_name_boost: 0.9 - - # For searching repositories at /api/repositories: - #repo_description_boost: 0.6 - - # For searching repositories at /api/repositories: - #repo_long_description_boost: 0.5 - - # For searching repositories at /api/repositories: - #repo_homepage_url_boost: 0.3 - - # For searching repositories at /api/repositories: - #repo_remote_repository_url_boost: 0.2 - - # For searching repositories at /api/repositories: - #repo_owner_username_boost: 0.3 - - # For searching repositories at /api/repositories: - #categories_boost: 0.5 - - # For searching tools at /api/tools - #tool_name_boost: 1.2 - - # For searching tools at /api/tools - #tool_description_boost: 0.6 - - # For searching tools at /api/tools - #tool_help_boost: 0.4 - - # For searching tools at /api/tools - #tool_repo_owner_username: 0.3 - - # You can enter tracking code here to track visitor's behavior through - # your Google Analytics account. Example: UA-XXXXXXXX-Y - #ga_code: null - - # The Tool Shed encodes various internal values when these values will - # be output in some format (for example, in a URL or cookie). You - # should set a key to be used by the algorithm that encodes and - # decodes these values. It can be any string. If left unchanged, - # anyone could construct a cookie that would grant them access to - # others' sessions. One simple way to generate a value for this is - # with the shell command: python -c 'from __future__ import - # print_function; import time; print(time.time())' | md5sum | cut -f 1 - # -d ' ' - #id_secret: changethisinproductiontoo - - # User authentication can be delegated to an upstream proxy server - # (usually Apache). The upstream proxy should set a REMOTE_USER - # header in the request. Enabling remote user disables regular logins. - # For more information, see: https://galaxyproject.org/admin/config - # /apache-external-user-auth/ - #use_remote_user: false - - # If use_remote_user is enabled, anyone who can log in to the Galaxy - # host may impersonate any other user by simply sending the - # appropriate header. Thus a secret shared between the upstream proxy - # server, and Galaxy is required. If anyone other than the Galaxy user - # is using the server, then apache/nginx should pass a value in the - # header 'GX_SECRET' that is identical the one below - #remote_user_secret: changethisinproductiontoo - - # If use_remote_user is enabled and your external authentication - # method just returns bare usernames, set a default mail domain to be - # appended to usernames, to become your Galaxy usernames (email - # addresses). - #remote_user_maildomain: null - - # If use_remote_user is enabled, the header that the upstream proxy - # provides the remote username in defaults to HTTP_REMOTE_USER (the - # 'HTTP_' is prepended by WSGI). This option allows you to change the - # header. Note, you still need to prepend 'HTTP_' to the header in - # this option, but your proxy server should *not* include 'HTTP_' at - # the beginning of the header name. - #remote_user_header: HTTP_REMOTE_USER - - # If use_remote_user is enabled, you can set this to a URL that will - # log your users out. - #remote_user_logout_href: null - - # Configuration for debugging middleware - #debug: false - - # Check for WSGI compliance. - #use_lint: false - - # Intercept print statements and show them on the returned page. - #use_printdebug: true - - # NEVER enable this on a public site (even test or QA) - #use_interactive: false - - # Administrative users - set this to a comma-separated list of valid - # Tool Shed users (email addresses). These users will have access to - # the Admin section of the server, and will have access to create - # users, groups, roles, libraries, and more. - #admin_users: null - - # Force everyone to log in (disable anonymous access) - #require_login: false - - # Allow unregistered users to create new accounts (otherwise, they - # will have to be created by an admin). - #allow_user_creation: true - - # Allow administrators to delete accounts. - #allow_user_deletion: false - - # For use by email messages sent from the Tool Shed. - #smtp_server: smtp.your_tool_shed_server - - # For use by email messages sent from the Tool Shed. - #email_from: your_tool_shed_email@server - - # If your SMTP server requires a username and password, you can - # provide them here (password in cleartext here, but if your server - # supports STARTTLS it will be sent over the network encrypted). - #smtp_username: null - - # If your SMTP server requires a username and password, you can - # provide them here (password in cleartext here, but if your server - # supports STARTTLS it will be sent over the network encrypted). - #smtp_password: null - - # If your SMTP server requires SSL from the beginning of the - # connection - #smtp_ssl: false - - # The URL linked by the "Support" link in the "Help" menu. - #support_url: https://galaxyproject.org/support/ - - # Address to join mailing list - #mailing_join_addr: galaxy-announce-join@bx.psu.edu - - # Write thread status periodically to 'heartbeat.log' (careful, uses - # disk space rapidly!) - #use_heartbeat: true - - # Profiling middleware (cProfile based) - #use_profile: true - - # Enable creation of Galaxy flavor Docker Image - #enable_galaxy_flavor_docker_image: false - - # Show a message box under the masthead. - #message_box_visible: false - - # Show a message box under the masthead. - #message_box_content: null - - # Class of the message box under the masthead. Possible values are: - # 'info' (the default), 'warning', 'error', 'done'. - #message_box_class: info - - # Serving static files (needed if running standalone) - #static_enabled: true - - # Serving static files (needed if running standalone) - #static_cache_time: 360 - - # Serving static files (needed if running standalone) - #static_dir: static/ - - # Serving static files (needed if running standalone) - #static_images_dir: static/images - - # Serving static files (needed if running standalone) - #static_favicon_dir: static/favicon.ico - - # Serving static files (needed if running standalone) - #static_scripts_dir: static/scripts/ - - # Serving static files (needed if running standalone) - #static_style_dir: static/style/blue - - # Enables GDPR Compliance mode. This makes several changes to the way - # Galaxy logs and exposes data externally such as removing - # emails/usernames from logs and bug reports. You are responsible for - # removing personal data from backups. Please read the GDPR section - # under the special topics area of the admin documentation. - #enable_beta_gdpr: false - - # For help on configuring the Advanced proxy features, see: - # https://docs.galaxyproject.org/en/master/admin/production.html - # Apache can handle file downloads (Galaxy-to-user) via mod_xsendfile. - # Set this to True to inform Galaxy that mod_xsendfile is enabled - # upstream. - #apache_xsendfile: false - - # The same download handling can be done by nginx using X-Accel- - # Redirect. This should be set to the path defined in the nginx - # config as an internal redirect with access to Galaxy's data files - # (see documentation linked above). - #nginx_x_accel_redirect_base: null - - # This value overrides the action set on the file upload form, e.g. - # the web path where the nginx_upload_module has been configured to - # intercept upload requests. - #nginx_upload_path: null - - # E-mail domains blacklist is used for filtering out users that are - # using disposable email address during the registration. If their - # address domain matches any domain in the blacklist, they are refused - # the registration. - #blacklist_file: config/disposable_email_blacklist.conf - - # Append "/{brand}" to the "Galaxy" text in the masthead. - #brand: null - - # Citation related caching. Tool citations information maybe fetched - # from external sources such as https://doi.org/ by Galaxy - the - # following parameters can be used to control the caching used to - # store this information. - #citation_cache_type: file - - # Citation related caching. Tool citations information maybe fetched - # from external sources such as https://doi.org/ by Galaxy - the - # following parameters can be used to control the caching used to - # store this information. - #citation_cache_data_dir: database/citations/data - - # Citation related caching. Tool citations information maybe fetched - # from external sources such as https://doi.org/ by Galaxy - the - # following parameters can be used to control the caching used to - # store this information. - #citation_cache_lock_dir: database/citations/lock - - # Turn on logging of user actions to the database. Actions currently - # logged are grid views, tool searches, and use of "recently" used - # tools menu. The log_events and log_actions functionality will - # eventually be merged. - #log_actions: true - - # Password expiration period (in days). Users are required to change - # their password every x days. Users will be redirected to the change - # password screen when they log in after their password expires. Enter - # 0 to disable password expiration. - #password_expiration_period: 0 - - # Log to Sentry Sentry is an open source logging and error aggregation - # platform. Setting sentry_dsn will enable the Sentry middleware and - # errors will be sent to the indicated sentry instance. This - # connection string is available in your sentry instance under - # -> Settings -> API Keys. - #sentry_dsn: null - - # Galaxy Session Timeout This provides a timeout (in minutes) after - # which a user will have to log back in. A duration of 0 disables this - # feature. - #session_duration: 0 - - # The URL linked by the "Terms and Conditions" link in the "Help" - # menu, as well as on the user registration and login forms and in the - # activation emails. - #terms_url: null - diff --git a/config/tool_shed.yml.sample b/config/tool_shed.yml.sample new file mode 120000 index 00000000000..02f3652c56d --- /dev/null +++ b/config/tool_shed.yml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/tool_shed.yml.sample \ No newline at end of file diff --git a/config/tool_sheds_conf.xml.sample b/config/tool_sheds_conf.xml.sample deleted file mode 100644 index 197cd63e384..00000000000 --- a/config/tool_sheds_conf.xml.sample +++ /dev/null @@ -1,7 +0,0 @@ - - - - - diff --git a/config/tool_sheds_conf.xml.sample b/config/tool_sheds_conf.xml.sample new file mode 120000 index 00000000000..5eb636b0370 --- /dev/null +++ b/config/tool_sheds_conf.xml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/tool_sheds_conf.xml.sample \ No newline at end of file diff --git a/config/user_preferences_extra_conf.yml.sample b/config/user_preferences_extra_conf.yml.sample deleted file mode 100644 index 2025f845e4b..00000000000 --- a/config/user_preferences_extra_conf.yml.sample +++ /dev/null @@ -1,65 +0,0 @@ -preferences: - # the key you can refer to - apollo_url_01: - # description that is displayed to the user - description: The URL to your personal Apollo instance - inputs: - # the content can be accessed by apollo_url_01.apollo_url - - name: apollo_url - label: Apollo URL - # type of input field that will be displayed to the user - # can be string or password - type: text - # by defaul all inputs are required - required: True - - name: apollo_text - label: Apollo Text - # type of input field that will be displayed to the user - # can be string or password - type: text - # by defaul all inputs are required - required: True - - openstack_account: - description: Your own Open Stack account - inputs: - - name: username - label: Username - type: text - required: False - - name: url - label: Open Stack URL - type: text - required: True - - webhook_overlay_search: - description: Configuration option for your search results - inputs: - - name: max_search_results - label: Maximum number of search results - type: text - required: False - - use_cached_job: - description: Do you want to be able to re-use previously run jobs ? - inputs: - - name: use_cached_job_checkbox - label: Do you want to be able to re-use equivalent jobs ? - type: boolean - checked: false - value: false - help: If you select yes, you will be able to select for each tool and workflow run if you would like to use this feature. - - localization: - description: Localization - inputs: - - name: locale - label: Prefered language - type: select - required: False - options: - - [Navigator default, auto] - - [Chinese, zh] - - [English, en] - - [French, fr] - - [Japanese, ja] diff --git a/config/user_preferences_extra_conf.yml.sample b/config/user_preferences_extra_conf.yml.sample new file mode 120000 index 00000000000..73884f1b7fe --- /dev/null +++ b/config/user_preferences_extra_conf.yml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/user_preferences_extra_conf.yml.sample \ No newline at end of file diff --git a/config/workflow_resource_mapper_conf.yml.sample b/config/workflow_resource_mapper_conf.yml.sample deleted file mode 100644 index 648dd09ddee..00000000000 --- a/config/workflow_resource_mapper_conf.yml.sample +++ /dev/null @@ -1,14 +0,0 @@ -by_group: - default: default - groups: - default: [project, priority] - prio_basic: [{name: priority, options: ["low", "med"]}] - prio_advanced: [{name: priority, options: ["low", "med", "high"]}] - prio_super: - - time - - memory - - processors - - name: priority - options: - - ultra - - plus_ultra diff --git a/config/workflow_resource_mapper_conf.yml.sample b/config/workflow_resource_mapper_conf.yml.sample new file mode 120000 index 00000000000..921cc9b39b7 --- /dev/null +++ b/config/workflow_resource_mapper_conf.yml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/workflow_resource_mapper_conf.yml.sample \ No newline at end of file diff --git a/config/workflow_resource_params_conf.xml.sample b/config/workflow_resource_params_conf.xml.sample deleted file mode 100644 index 0495c2443ed..00000000000 --- a/config/workflow_resource_params_conf.xml.sample +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - diff --git a/config/workflow_resource_params_conf.xml.sample b/config/workflow_resource_params_conf.xml.sample new file mode 120000 index 00000000000..1b4ac008309 --- /dev/null +++ b/config/workflow_resource_params_conf.xml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/workflow_resource_params_conf.xml.sample \ No newline at end of file diff --git a/config/workflow_schedulers_conf.xml.sample b/config/workflow_schedulers_conf.xml.sample deleted file mode 100644 index 1c83136f870..00000000000 --- a/config/workflow_schedulers_conf.xml.sample +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - - - - - - diff --git a/config/workflow_schedulers_conf.xml.sample b/config/workflow_schedulers_conf.xml.sample new file mode 120000 index 00000000000..000190a25de --- /dev/null +++ b/config/workflow_schedulers_conf.xml.sample @@ -0,0 +1 @@ +../lib/galaxy/config/sample/workflow_schedulers_conf.xml.sample \ No newline at end of file diff --git a/doc/Makefile b/doc/Makefile index c4589e50479..644c424d8a0 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -57,7 +57,7 @@ source/api/ts_api.rst: source/lib/galaxy.webapps.tool_shed.api.rst sed -i.bak -e 's/^galaxy\\\.webapps\\\.tool\\_shed\\\.api\\\.//' $@ rm -f $@.bak -source/admin/config_logging_default_yaml.rst: ../lib/galaxy/config.py +source/admin/config_logging_default_yaml.rst: ../lib/galaxy/config/__init__.py printf '.. code-block:: yaml\n\n' > $@ printf ' galaxy:\n' >> $@ printf ' logging:\n ' >> $@ diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 30f04e85099..398ca602f96 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -222,7 +222,7 @@ functionality is largely untested in modern Galaxy releases and has serious issues such as #7273 and the possibility of slowing down Galaxy startup, so the default and recommended value is - False. + false. :Default: ``false`` :Type: bool @@ -271,14 +271,13 @@ ~~~~~~~~~~~~~~~~~~~~~~~ :Description: - Path to the directory in which tool dependencies are placed. This - is used by the Tool Shed to install dependencies and can also be - used by administrators to manually install or link to - dependencies. For details, see: - https://galaxyproject.org/admin/config/tool-dependencies Set the + Various dependency resolver configuration parameters will have + defaults set relative to this path, such as the default conda + prefix, default Galaxy packages path, legacy tool shed + dependencies path, and the dependency cache directory. Set the string to None to explicitly disable tool dependency handling. If this option is set to none or an invalid path, installing tools - with dependencies from the Tool Shed will fail. + with dependencies from the Tool Shed or in Conda will fail. :Default: ``database/dependencies`` :Type: str @@ -360,7 +359,7 @@ ~~~~~~~~~~~~~~~~~~~~~~ :Description: - Set to True to instruct Galaxy to look for and install missing + Set to true to instruct Galaxy to look for and install missing tool dependencies before each job runs. :Default: ``false`` :Type: bool @@ -371,7 +370,7 @@ ~~~~~~~~~~~~~~~~~~~ :Description: - Set to True to instruct Galaxy to install Conda from the web + Set to true to instruct Galaxy to install Conda from the web automatically if it cannot find a local copy and conda_exec is not configured. :Default: ``true`` @@ -383,7 +382,7 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~ :Description: - You must set this to True if conda_prefix and + You must set this to true if conda_prefix and job_working_directory are not on the same volume, or some conda dependencies will fail to execute at job runtime. Conda will copy packages content instead of creating hardlinks or symlinks. This @@ -402,10 +401,12 @@ Certain dependency resolvers (namely Conda) take a considerable amount of time to build an isolated job environment in the job_working_directory if the job working directory is on a network - share. Set the following option to True to cache the dependencies - in a folder. This option is beta and should only be used if you + share. Set this option to true to cache the dependencies in a + folder. This option is beta and should only be used if you experience long waiting times before a job is actually submitted - to your cluster. + to your cluster. This only affects tools where some requirements + can be resolved but not others, most modern best practice tools + can use prebuilt environments in the Conda directory. :Default: ``false`` :Type: bool @@ -428,7 +429,7 @@ :Description: By default, when using a cached dependency manager, the dependencies are cached when installing new tools and when using - tools for the first time. Set this to False if you prefer + tools for the first time. Set this to false if you prefer dependencies to be cached only when installing new tools. :Default: ``true`` :Type: bool @@ -451,7 +452,7 @@ ~~~~~~~~~~~~~~~ :Description: - Set to True to enable monitoring of tools and tool directories + Set to true to enable monitoring of tools and tool directories listed in any tool config file specified in tool_config_file option. If changes are found, tools are automatically reloaded. Watchdog ( https://pypi.org/project/watchdog/ ) must be installed @@ -469,7 +470,7 @@ ~~~~~~~~~~~~~~~~~~~ :Description: - Set to True to enable monitoring of dynamic job rules. If changes + Set to true to enable monitoring of dynamic job rules. If changes are found, rules are automatically reloaded. Takes the same values as the 'watch_tools' option. :Default: ``false`` @@ -535,7 +536,7 @@ `requirement`s. The following path is the location of involucro on the Galaxy host. This is ignored if the relevant container resolver isn't enabled, and will install on demand unless - involucro_auto_init is set to False. + involucro_auto_init is set to false. :Default: ``database/dependencies/involucro`` :Type: str @@ -660,7 +661,7 @@ ~~~~~~~~~~~~~~~~~~~~~~~ :Description: - Set to True to enable monitoring of the tool_data and + Set to true to enable monitoring of the tool_data and shed_tool_data_path directories. If changes in tool data table files are found, the tool data tables for that data manager are automatically reloaded. Watchdog ( @@ -716,7 +717,7 @@ :Description: Enable sniffing of compressed datatypes. This can be configured/overridden on a per-datatype basis in the - datatypes_conf.xml file. With this option set to False the + datatypes_conf.xml file. With this option set to false the compressed datatypes will be unpacked before sniffing. :Default: ``true`` :Type: bool @@ -767,7 +768,7 @@ :Description: To run interactive environment containers in Docker Swarm mode (on - an existing swarm), set this option to True and set + an existing swarm), set this option to true and set `docker_connect_port` in the IE plugin config (ini) file(s) of any IE plugins you have enabled and ensure that you are not using any `docker run`-specific options in your plugins' `command_inject` @@ -1099,9 +1100,9 @@ ~~~~~~~~~~~~~~~~~~~~~~ :Description: - User account activation feature global flag. If set to "False", - the rest of the Account activation configuration is ignored and - user activation is disabled (i.e. accounts are active since + User account activation feature global flag. If set to false, the + rest of the Account activation configuration is ignored and user + activation is disabled (i.e. accounts are active since registration). The activation is also not working in case the SMTP server is not defined. :Default: ``false`` @@ -1176,9 +1177,9 @@ Galaxy can display data at various external browsers. These options specify which browsers should be available. URLs and builds available at these browsers are defined in the specified - files. If use_remote_user = True, display application servers - will be denied access to Galaxy and so displaying datasets in - these sites will fail. display_servers contains a list of + files. If use_remote_user is set to true, display application + servers will be denied access to Galaxy and so displaying datasets + in these sites will fail. display_servers contains a list of hostnames which should be allowed to bypass security to display datasets. Please be aware that there are security implications if this is allowed. More details (including required changes to the @@ -1197,15 +1198,15 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :Description: - To disable the old-style display applications that are hardcoded - into datatype classes, set enable_old_display_applications = - False. This may be desirable due to using the new-style, XML- - defined, display applications that have been defined for many of - the datatypes that have the old-style. There is also a potential - security concern with the old-style applications, where a - malicious party could provide a link that appears to reference the - Galaxy server, but contains a redirect to a third-party server, - tricking a Galaxy user to access said site. + Set this to false to disable the old-style display applications + that are hardcoded into datatype classes. This may be desirable + due to using the new-style, XML-defined, display applications that + have been defined for many of the datatypes that have the old- + style. There is also a potential security concern with the old- + style applications, where a malicious party could provide a link + that appears to reference the Galaxy server, but contains a + redirect to a third-party server, tricking a Galaxy user to access + said site. :Default: ``true`` :Type: bool @@ -1576,7 +1577,7 @@ For help on configuring the Advanced proxy features, see: https://docs.galaxyproject.org/en/master/admin/production.html Apache can handle file downloads (Galaxy-to-user) via - mod_xsendfile. Set this to True to inform Galaxy that + mod_xsendfile. Set this to true to inform Galaxy that mod_xsendfile is enabled upstream. :Default: ``false`` :Type: bool @@ -1697,9 +1698,9 @@ other services based on Galaxy's session cookie. It will attempt to do this by default though you do need to install node+npm and do an npm install from `lib/galaxy/web/proxy/js`. It is generally - more robust to configure this externally, managing it however - Galaxy is managed. If True, Galaxy will only launch the proxy if - it is actually going to be used (e.g. for Jupyter). + more robust to configure this externally, managing it in the same + way Galaxy itself is managed. If true, Galaxy will only launch + the proxy if it is actually going to be used (e.g. for Jupyter). :Default: ``true`` :Type: bool @@ -1733,7 +1734,8 @@ :Description: Set the port and IP for the dynamic proxy to bind to, this must - match the external configuration if dynamic_proxy_manage is False. + match the external configuration if dynamic_proxy_manage is set to + false. :Default: ``8800`` :Type: int @@ -1744,7 +1746,8 @@ :Description: Set the port and IP for the dynamic proxy to bind to, this must - match the external configuration if dynamic_proxy_manage is False. + match the external configuration if dynamic_proxy_manage is set to + false. :Default: ``0.0.0.0`` :Type: str @@ -1837,7 +1840,7 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~ :Description: - If True, Galaxy will attempt to configure a simple root logger if + If true, Galaxy will attempt to configure a simple root logger if a "loggers" section does not appear in this configuration file. :Default: ``true`` :Type: bool @@ -1984,8 +1987,7 @@ By default Galaxy will serve non-HTML tool output that may potentially contain browser executable JavaScript content as plain text. This will for instance cause SVG datasets to not render - properly and so may be disabled by setting the following option to - True. + properly and so may be disabled by setting this option to true. :Default: ``false`` :Type: bool @@ -2011,12 +2013,11 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :Description: - Set the following to True to use Jupyter nbconvert to build HTML - from Jupyter notebooks in Galaxy histories. This process may - allow users to execute arbitrary code or serve arbitrary HTML. If - enabled, Jupyter must be available and on Galaxy's PATH, to do - this run `pip install jinja2 pygments jupyter` in Galaxy's - virtualenv. + Set to true to use Jupyter nbconvert to build HTML from Jupyter + notebooks in Galaxy histories. This process may allow users to + execute arbitrary code or serve arbitrary HTML. If enabled, + Jupyter must be available and on Galaxy's PATH, to do this run + `pip install jinja2 pygments jupyter` in Galaxy's virtualenv. :Default: ``false`` :Type: bool @@ -2294,7 +2295,7 @@ allowing admins to paste filesystem paths to files and directories in a box, and these paths will be added to a library. For history uploads, this allows pasting in paths as URIs. (i.e. prefixed with - file://). Set to True to enable. Please note the security + file://). Set to true to enable. Please note the security implication that this will give Galaxy Admins access to anything your Galaxy user has access to. :Default: ``false`` @@ -2559,8 +2560,8 @@ :Description: If your proxy and/or authentication source does not normalize - e-mail addresses or user names being passed to Galaxy - set the - following option to True to force these to lower case. + e-mail addresses or user names being passed to Galaxy - set this + option to true to force these to lower case. :Default: ``false`` :Type: bool @@ -2609,7 +2610,7 @@ :Description: Show the site's welcome page (see welcome_url) alongside the login - page (even if require_login is True) + page (even if require_login is true). :Default: ``false`` :Type: bool @@ -2641,7 +2642,7 @@ :Description: Allow administrators to log in as other users (useful for - debugging) + debugging). :Default: ``false`` :Type: bool @@ -2674,7 +2675,7 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :Description: - By default, users' data will be public, but setting this to True + By default, users' data will be public, but setting this to true will cause it to be private. Does not affect existing users and data, only ones created after this option is set. Users may still change their default back to public. @@ -2687,14 +2688,14 @@ ~~~~~~~~~~~~~~~~~~~~ :Description: - Expose user list. Setting this to True will expose the user list + Expose user list. Setting this to true will expose the user list to authenticated users. This makes sharing datasets in smaller galaxy instances much easier as they can type a name/email and have the correct user show up. This makes less sense on large public Galaxy instances where that data shouldn't be exposed. For semi-public Galaxies, it may make sense to expose just the username and not email, or vice versa. If enable_beta_gdpr is set - to True, then this option will be overridden and set to False. + to true, then this option will be overridden and set to false. :Default: ``false`` :Type: bool @@ -2704,14 +2705,14 @@ ~~~~~~~~~~~~~~~~~~~~~ :Description: - Expose user list. Setting this to True will expose the user list + Expose user list. Setting this to true will expose the user list to authenticated users. This makes sharing datasets in smaller galaxy instances much easier as they can type a name/email and have the correct user show up. This makes less sense on large public Galaxy instances where that data shouldn't be exposed. For semi-public Galaxies, it may make sense to expose just the username and not email, or vice versa. If enable_beta_gdpr is set - to True, then this option will be overridden and set to False. + to true, then this option will be overridden and set to false. :Default: ``false`` :Type: bool @@ -2753,17 +2754,6 @@ :Type: bool -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``enable_beta_ts_api_install`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:Description: - Enable the new interface for installing tools from Tool Shed via - the API. Admin menu will list both if enabled. -:Default: ``true`` -:Type: bool - - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``enable_beta_containers_interface`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -3000,9 +2990,9 @@ datasets are selected for "Set at Runtime" inputs from the history such that the same input will not be selected twice, unless there are more inputs than compatible datasets in the history. When - False, the most recently added compatible item in the history will + false, the most recently added compatible item in the history will be used for each "Set at Runtime" input, independent of others in - the Workflow + the workflow. :Default: ``false`` :Type: bool @@ -3013,7 +3003,7 @@ :Description: The URL to the myExperiment instance being used (omit scheme but - include port) + include port). :Default: ``www.myexperiment.org:80`` :Type: str @@ -3073,8 +3063,8 @@ ~~~~~~~~~~~~~~~~~~~~ :Description: - This should be set to False to prevent Galaxy from deleting - uploaded FTP files as it imports them. + Set to false to prevent Galaxy from deleting uploaded FTP files as + it imports them. :Default: ``true`` :Type: bool @@ -3210,6 +3200,20 @@ :Type: seq +~~~~~~~~~~~~~~~~~~~~~~~~~ +``dependency_resolution`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Alternative representation of various dependency resolution + parameters. Takes the dictified version of a DependencyManager + object - so this is ideal for automating the configuration of + dependency resolution from one application that uses a + DependencyManager to another. +:Default: ``None`` +:Type: map + + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``default_job_resubmission_condition`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -3290,7 +3294,7 @@ fail. In these instances, you can choose to retry setting it internally or leave it in a failed state (since retrying internally may cause the Galaxy process to be unresponsive). If - this option is set to False, the user will be given the option to + this option is set to false, the user will be given the option to retry externally, or set metadata manually (when possible). :Default: ``true`` :Type: bool @@ -3510,7 +3514,7 @@ several extra database queries must be performed to determine the number of jobs a user has dispatched to a given destination. By default, these queries will happen for every job that is waiting - to run, but if cache_user_job_count is set to True, it will only + to run, but if cache_user_job_count is set to true, it will only happen once per iteration of the handler queue. Although better for performance due to reduced queries, the trade-off is a greater possibility that jobs will be dispatched past the configured diff --git a/doc/source/lib/log_tempfile.rst b/doc/source/lib/log_tempfile.rst deleted file mode 100644 index a1dfd15db15..00000000000 --- a/doc/source/lib/log_tempfile.rst +++ /dev/null @@ -1,7 +0,0 @@ -log\_tempfile module -==================== - -.. automodule:: log_tempfile - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/source/lib/mimeparse.rst b/doc/source/lib/mimeparse.rst deleted file mode 100644 index afcdfa0c86b..00000000000 --- a/doc/source/lib/mimeparse.rst +++ /dev/null @@ -1,7 +0,0 @@ -mimeparse module -================ - -.. automodule:: mimeparse - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/source/lib/modules.rst b/doc/source/lib/modules.rst index fc89779cef0..448752bc959 100644 --- a/doc/source/lib/modules.rst +++ b/doc/source/lib/modules.rst @@ -6,7 +6,4 @@ lib galaxy galaxy_ext - log_tempfile - mimeparse - psyco_full tool_shed diff --git a/doc/source/lib/psyco_full.rst b/doc/source/lib/psyco_full.rst deleted file mode 100644 index 8be871ed767..00000000000 --- a/doc/source/lib/psyco_full.rst +++ /dev/null @@ -1,7 +0,0 @@ -psyco\_full module -================== - -.. automodule:: psyco_full - :members: - :undoc-members: - :show-inheritance: diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 03503ccb6a9..ce95e3f1d92 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -11,6 +11,7 @@ import galaxy.queues import galaxy.quota import galaxy.security from galaxy import config, job_metrics, jobs +from galaxy.config_watchers import ConfigWatchers from galaxy.containers import build_container_interfaces from galaxy.managers.collections import DatasetCollectionManager from galaxy.managers.folders import FolderManager @@ -41,7 +42,6 @@ from galaxy.visualization.plugins.registry import VisualizationsRegistry from galaxy.web import url_for from galaxy.web.proxy import ProxyManager from galaxy.web_stack import application_stack_instance -from galaxy.webapps.galaxy.config_watchers import ConfigWatchers from galaxy.webhooks import WebhooksRegistry from tool_shed.galaxy_install import ( installed_repository_manager, diff --git a/lib/galaxy/auth/__init__.py b/lib/galaxy/auth/__init__.py index 1857335f004..470f7667fef 100644 --- a/lib/galaxy/auth/__init__.py +++ b/lib/galaxy/auth/__init__.py @@ -15,7 +15,7 @@ class AuthManager(object): def __init__(self, app): self.__app = app self.redact_username_in_logs = app.config.redact_username_in_logs - self.authenticators = get_authenticators(app.config.auth_config_file) + self.authenticators = get_authenticators(app.config.auth_config_file, app.config.auth_config_file_set) def check_registration_allowed(self, email, username, password): """Checks if the provided email/username is allowed to register.""" diff --git a/lib/galaxy/auth/util.py b/lib/galaxy/auth/util.py index b256fc29ec2..86bec13c692 100644 --- a/lib/galaxy/auth/util.py +++ b/lib/galaxy/auth/util.py @@ -1,3 +1,4 @@ +import errno import logging import xml.etree.ElementTree from collections import namedtuple @@ -10,14 +11,31 @@ from galaxy.util import plugin_config, string_as_bool log = logging.getLogger(__name__) +AUTH_CONF_XML = """ + + + localdb + + true + + + +""" + Authenticator = namedtuple('Authenticator', ['plugin', 'filter_template', 'options']) -def get_authenticators(auth_config_file): +def get_authenticators(auth_config_file, auth_config_file_set): __plugins_dict = plugin_config.plugins_dict(galaxy.auth.providers, 'plugin_type') # parse XML - ct = xml.etree.ElementTree.parse(auth_config_file) - conf_root = ct.getroot() + try: + ct = xml.etree.ElementTree.parse(auth_config_file) + conf_root = ct.getroot() + except (OSError, IOError) as exc: + if exc.errno == errno.ENOENT and not auth_config_file_set: + conf_root = xml.etree.ElementTree.fromstring(AUTH_CONF_XML) + else: + raise authenticators = [] # process authenticators diff --git a/lib/galaxy/config.py b/lib/galaxy/config/__init__.py similarity index 84% rename from lib/galaxy/config.py rename to lib/galaxy/config/__init__.py index ec15e50698b..851e657a00f 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config/__init__.py @@ -5,6 +5,7 @@ Universe configuration builder. from __future__ import absolute_import import collections +import errno import ipaddress import logging import logging.config @@ -25,6 +26,8 @@ from six.moves import configparser from galaxy.containers import parse_containers_config from galaxy.exceptions import ConfigurationError +from galaxy.model import mapping +from galaxy.model.tool_shed_install.migrate.check import create_or_verify_database as tsi_create_or_verify_database from galaxy.tool_util.deps.container_resolvers.mulled import DEFAULT_CHANNELS from galaxy.util import ( ExecutionTimer, @@ -34,59 +37,16 @@ from galaxy.util import ( ) from galaxy.util.dbkeys import GenomeBuilds from galaxy.util.logging import LOGLV_TRACE +from galaxy.util.properties import find_config_file, running_from_source from galaxy.web.formatting import expand_pretty_datetime_format from galaxy.web_stack import ( get_stack_facts, register_postfork_function ) -from .version import VERSION_MAJOR +from ..version import VERSION_MAJOR log = logging.getLogger(__name__) - -PATH_DEFAULTS = dict( - auth_config_file=['config/auth_conf.xml', 'config/auth_conf.xml.sample'], - data_manager_config_file=['config/data_manager_conf.xml', 'data_manager_conf.xml', 'config/data_manager_conf.xml.sample'], - datatypes_config_file=['config/datatypes_conf.xml', 'datatypes_conf.xml', 'config/datatypes_conf.xml.sample'], - build_sites_config_file=['config/build_sites.yml', 'config/build_sites.yml.sample'], - job_config_file=['config/job_conf.xml', 'job_conf.xml'], - tool_destinations_config_file=['config/tool_destinations.yml', 'config/tool_destinations.yml.sample'], - job_metrics_config_file=['config/job_metrics_conf.xml', 'job_metrics_conf.xml', 'config/job_metrics_conf.xml.sample'], - error_report_file=['config/error_report.yml', 'config/error_report.yml.sample'], - oidc_config_file=['config/oidc_config.yml', 'config/oidc_config.yml.sample'], - oidc_backends_config_file=['config/oidc_backends_config.yml', 'config/oidc_backends_config.yml.sample'], - dependency_resolvers_config_file=['config/dependency_resolvers_conf.xml', 'dependency_resolvers_conf.xml', None], - job_resource_params_file=['config/job_resource_params_conf.xml', 'job_resource_params_conf.xml'], - workflow_resource_params_file=['config/workflow_resource_params_conf.xml', 'workflow_resource_params_conf.xml'], - migrated_tools_config=['migrated_tools_conf.xml', 'config/migrated_tools_conf.xml'], - object_store_config_file=['config/object_store_conf.xml', 'object_store_conf.xml'], - shed_data_manager_config_file=['shed_data_manager_conf.xml', 'config/shed_data_manager_conf.xml'], - shed_tool_data_table_config=['shed_tool_data_table_conf.xml', 'config/shed_tool_data_table_conf.xml'], - tool_sheds_config_file=['config/tool_sheds_conf.xml', 'tool_sheds_conf.xml', 'config/tool_sheds_conf.xml.sample'], - workflow_schedulers_config_file=['config/workflow_schedulers_conf.xml', 'config/workflow_schedulers_conf.xml.sample'], - modules_mapping_files=['config/environment_modules_mapping.yml', 'config/environment_modules_mapping.yml.sample'], - local_conda_mapping_file=['config/local_conda_mapping.yml', 'config/local_conda_mapping.yml.sample'], - containers_config_file=['config/containers_conf.yml'], -) - -PATH_LIST_DEFAULTS = dict( - tool_data_table_config_path=['config/tool_data_table_conf.xml', 'tool_data_table_conf.xml', 'config/tool_data_table_conf.xml.sample'], - # rationale: - # [0]: user has explicitly created config/tool_conf.xml but did not - # move their existing shed_tool_conf.xml, don't use - # config/shed_tool_conf.xml, which is probably the empty - # version copied from the sample, or else their shed tools - # will disappear - # [1]: user has created config/tool_conf.xml and, having passed - # [0], probably moved their shed_tool_conf.xml as well - # [2]: user has done nothing, use the old files - # [3]: fresh install - tool_config_file=['config/tool_conf.xml,shed_tool_conf.xml', - 'config/tool_conf.xml,config/shed_tool_conf.xml', - 'tool_conf.xml,shed_tool_conf.xml', - 'config/tool_conf.xml.sample,config/shed_tool_conf.xml'] -) - LOGGING_CONFIG_DEFAULT = { 'version': 1, 'root': { @@ -139,40 +99,112 @@ def resolve_path(path, root): return path -def find_path(kwargs, var, root): - """Find a configuration path that may exist at different defaults.""" - defaults = PATH_DEFAULTS[var] - - if kwargs.get(var, None) is not None: - path = kwargs.get(var) - else: - for default in defaults: - if default is None: - # if None is the final default - just return that. - return None - if os.path.exists(resolve_path(default, root)): - path = default - break - else: - path = defaults[-1] - - return resolve_path(path, root) - - def find_root(kwargs): root = kwargs.get('root_dir', '.') return root -class Configuration(object): +class BaseAppConfiguration(object): + def _set_config_base(self, config_kwargs): + self.sample_config_dir = os.path.join(os.path.dirname(__file__), 'sample') + self.config_file = find_config_file('galaxy') + # Parse global_conf and save the parser + self.global_conf = config_kwargs.get('global_conf', None) + self.global_conf_parser = configparser.ConfigParser() + if not self.config_file and self.global_conf and "__file__" in self.global_conf: + self.config_file = self.global_conf['__file__'] + if self.config_file is None: + log.warning("No Galaxy config file found, running from current working directory: %s", os.getcwd()) + else: + try: + self.global_conf_parser.read(self.config_file) + except (IOError, OSError): + raise + except Exception: + # Not an INI file + pass + self.config_dir = config_kwargs.get('config_dir', os.path.dirname(self.config_file or os.getcwd())) + self.data_dir = config_kwargs.get('data_dir', None) + # mutable_config_dir is intentionally not configurable. You can + # override individual mutable configs with config options, but they + # should be considered Galaxy-controlled data files and will by default + # just live in the data dir + if running_from_source: + if self.data_dir is None: + self.data_dir = os.path.join(self.root, 'database') + if self.config_file is None: + self.config_dir = os.path.join(self.root, 'config') + self.mutable_config_dir = self.config_dir + # TODO: do we still need to support ../shed_tools? + self.shed_tools_dir = os.path.join(self.data_dir, 'shed_tools') + else: + if self.data_dir is None: + self.data_dir = os.path.join(self.config_dir, 'data') + self.mutable_config_dir = os.path.join(self.data_dir, 'config') + self.shed_tools_dir = os.path.join(self.data_dir, 'shed_tools') + log.debug("Configuration directory is %s", self.config_dir) + log.debug("Data directory is %s", self.data_dir) + log.debug("Mutable config directory is %s", self.mutable_config_dir) + + def _in_mutable_config_dir(self, path): + return os.path.join(self.mutable_config_dir, path) + + def _in_config_dir(self, path): + return os.path.join(self.config_dir, path) + + def _in_sample_dir(self, path): + return os.path.join(self.sample_config_dir, path) + + def _parse_config_file_options(self, defaults, listify_defaults, config_kwargs): + for var, defaults in defaults.items(): + if config_kwargs.get(var, None) is not None: + path = config_kwargs.get(var) + setattr(self, var + '_set', True) + else: + for default in defaults: + if os.path.exists(resolve_path(default, self.root)): + path = default + break + else: + path = defaults[-1] + setattr(self, var + '_set', False) + setattr(self, var, resolve_path(path, self.root)) + + for var, defaults in listify_defaults.items(): + paths = [] + if config_kwargs.get(var, None) is not None: + paths = listify(config_kwargs.get(var)) + else: + for default in defaults: + for path in listify(default): + if not os.path.exists(resolve_path(path, self.root)): + break + else: + paths = listify(default) + break + else: + paths = listify(defaults[-1]) + setattr(self, var, [resolve_path(x, self.root) for x in paths]) + + +class GalaxyAppConfiguration(BaseAppConfiguration): deprecated_options = ('database_file', 'track_jobs_in_database') + default_config_file_name = 'galaxy.yml' def __init__(self, **kwargs): self.config_dict = kwargs self.root = find_root(kwargs) + self._set_config_base(kwargs) + + # Configs no longer read from samples + self.migrated_tools_config = resolve_path(kwargs.get('migrated_tools_conf', 'migrated_tools_conf.xml'), self.mutable_config_dir) + self.shed_tool_conf = resolve_path(kwargs.get('shed_tool_conf', 'shed_tool_conf.xml'), self.mutable_config_dir) + for name in ('migrated_tools_config', 'shed_tool_conf'): + setattr(self, name + '_set', kwargs.get(name, None) is not None) + # Resolve paths of other config files - self.__parse_config_file_options(kwargs) + self.parse_config_file_options(kwargs) # Collect the umask and primary gid from the environment self.umask = os.umask(0o77) # get the current umask @@ -182,8 +214,8 @@ class Configuration(object): self.version_major = VERSION_MAJOR # Database related configuration self.check_migrate_databases = kwargs.get('check_migrate_databases', True) - self.database = resolve_path(kwargs.get("database_file", "database/universe.sqlite"), self.root) - self.database_connection = kwargs.get("database_connection", False) + self.database_connection = kwargs.get("database_connection", + "sqlite:///%s?isolation_level=IMMEDIATE" % resolve_path("universe.sqlite", self.data_dir)) self.database_engine_options = get_database_engine_options(kwargs) self.database_create_tables = string_as_bool(kwargs.get("database_create_tables", "True")) self.database_query_profiling_proxy = string_as_bool(kwargs.get("database_query_profiling_proxy", "False")) @@ -210,19 +242,21 @@ class Configuration(object): self.database_wait_sleep = float(kwargs.get("database_wait_sleep", 1)) # Where dataset files are stored - self.file_path = resolve_path(kwargs.get("file_path", "database/files"), self.root) + self.file_path = resolve_path(kwargs.get("file_path", "files"), self.data_dir) # new_file_path and legacy_home_dir can be overridden per destination in job_conf. - self.new_file_path = resolve_path(kwargs.get("new_file_path", "database/tmp"), self.root) + self.new_file_path = resolve_path(kwargs.get("new_file_path", "tmp"), self.data_dir) override_tempdir = string_as_bool(kwargs.get("override_tempdir", "True")) if override_tempdir: tempfile.tempdir = self.new_file_path self.shared_home_dir = kwargs.get("shared_home_dir", None) - self.openid_consumer_cache_path = resolve_path(kwargs.get("openid_consumer_cache_path", "database/openid_consumer_cache"), self.root) + self.openid_consumer_cache_path = resolve_path(kwargs.get("openid_consumer_cache_path", "openid_consumer_cache"), self.data_dir) self.cookie_path = kwargs.get("cookie_path", None) self.enable_quotas = string_as_bool(kwargs.get('enable_quotas', False)) self.enable_unique_workflow_defaults = string_as_bool(kwargs.get('enable_unique_workflow_defaults', False)) self.tool_path = resolve_path(kwargs.get("tool_path", "tools"), self.root) self.tool_data_path = resolve_path(kwargs.get("tool_data_path", "tool-data"), os.getcwd()) + if not running_from_source and kwargs.get("tool_data_path", None) is None: + self.tool_data_path = resolve_path("tool-data", self.data_dir) self.builds_file_path = resolve_path(kwargs.get("builds_file_path", os.path.join(self.tool_data_path, 'shared', 'ucsc', 'builds.txt')), self.root) self.len_file_path = resolve_path(kwargs.get("len_file_path", os.path.join(self.tool_data_path, 'shared', 'ucsc', 'chrom')), self.root) # Galaxy OIDC settings. @@ -231,8 +265,8 @@ class Configuration(object): self.oidc_backends_config = kwargs.get("oidc_backends_config_file", self.oidc_backends_config_file) self.oidc = [] # The value of migrated_tools_config is the file reserved for containing only those tools that have been eliminated from the distribution - # and moved to the tool shed. - self.integrated_tool_panel_config = resolve_path(kwargs.get('integrated_tool_panel_config', 'integrated_tool_panel.xml'), self.root) + # and moved to the tool shed. It is created on demand. + self.integrated_tool_panel_config = resolve_path(kwargs.get('integrated_tool_panel_config', 'integrated_tool_panel.xml'), self.mutable_config_dir) integrated_tool_panel_tracking_directory = kwargs.get('integrated_tool_panel_tracking_directory', None) if integrated_tool_panel_tracking_directory: self.integrated_tool_panel_tracking_directory = resolve_path(integrated_tool_panel_tracking_directory, self.root) @@ -317,13 +351,14 @@ class Configuration(object): self.show_user_prepopulate_form = string_as_bool(kwargs.get("show_user_prepopulate_form", "False")) self.new_user_dataset_access_role_default_private = string_as_bool(kwargs.get("new_user_dataset_access_role_default_private", "False")) self.template_path = resolve_path(kwargs.get("template_path", "templates"), self.root) - self.template_cache = resolve_path(kwargs.get("template_cache_path", "database/compiled_templates"), self.root) + self.template_cache = resolve_path(kwargs.get("template_cache_path", "compiled_templates"), self.data_dir) self.job_queue_cleanup_interval = int(kwargs.get("job_queue_cleanup_interval", "5")) self.cluster_files_directory = os.path.abspath(kwargs.get("cluster_files_directory", "database/pbs")) + self.cluster_files_directory = os.path.abspath(resolve_path(kwargs.get("cluster_files_directory", "pbs"), self.data_dir)) # Fall back to legacy job_working_directory config variable if set. - default_jobs_directory = kwargs.get("job_working_directory", "database/jobs_directory") - self.jobs_directory = resolve_path(kwargs.get("jobs_directory", default_jobs_directory), self.root) + default_jobs_directory = kwargs.get("job_working_directory", "jobs_directory") + self.jobs_directory = resolve_path(kwargs.get("jobs_directory", default_jobs_directory), self.data_dir) self.default_job_shell = kwargs.get("default_job_shell", "/bin/bash") self.cleanup_job = kwargs.get("cleanup_job", "always") preserve_python_environment = kwargs.get("preserve_python_environment", "legacy_only") @@ -334,7 +369,7 @@ class Configuration(object): self.nodejs_path = kwargs.get("nodejs_path", None) # Older default container cache path, I don't think anyone is using it anymore and it wasn't documented - we # should probably drop the backward compatiblity to save the path check. - self.container_image_cache_path = self.resolve_path(kwargs.get("container_image_cache_path", "database/container_images")) + self.container_image_cache_path = resolve_path(kwargs.get("container_image_cache_path", "container_images"), self.data_dir) if not os.path.exists(self.container_image_cache_path): self.container_image_cache_path = self.resolve_path(kwargs.get("container_image_cache_path", "database/container_cache")) self.outputs_to_working_directory = string_as_bool(kwargs.get('outputs_to_working_directory', False)) @@ -385,10 +420,6 @@ class Configuration(object): # Tasked job runner. self.use_tasked_jobs = string_as_bool(kwargs.get('use_tasked_jobs', False)) self.local_task_queue_workers = int(kwargs.get("local_task_queue_workers", 2)) - - # Enable new interface for API installations from TS. - # Admin menu will list both if enabled. - self.enable_beta_ts_api_install = string_as_bool(kwargs.get('enable_beta_ts_api_install', 'True')) # The transfer manager and deferred job queue self.enable_beta_job_managers = string_as_bool(kwargs.get('enable_beta_job_managers', 'False')) # Set this to go back to setting the object store in the tool request instead of @@ -525,7 +556,7 @@ class Configuration(object): involucro_path = kwargs.get('involucro_path', None) if involucro_path is None: - target_dir = kwargs.get("tool_dependency_dir", "database/dependencies") + target_dir = resolve_path(kwargs.get("tool_dependency_dir", "dependencies"), self.data_dir) if target_dir == "none": target_dir = "database" involucro_path = os.path.join(target_dir, "involucro") @@ -554,7 +585,7 @@ class Configuration(object): self.nginx_upload_store = os.path.abspath(self.nginx_upload_store) self.object_store = kwargs.get('object_store', 'disk') self.object_store_check_old_style = string_as_bool(kwargs.get('object_store_check_old_style', False)) - self.object_store_cache_path = resolve_path(kwargs.get("object_store_cache_path", "database/object_store_cache"), self.root) + self.object_store_cache_path = resolve_path(kwargs.get("object_store_cache_path", "database/object_store_cache"), self.data_dir) self.object_store_store_by = kwargs.get("object_store_store_by", "id") # Handle AWS-specific config options for backward compatibility @@ -578,17 +609,9 @@ class Configuration(object): self.distributed_object_store_config_file = resolve_path(self.distributed_object_store_config_file, self.root) self.irods_root_collection_path = kwargs.get('irods_root_collection_path', None) self.irods_default_resource = kwargs.get('irods_default_resource', None) - # Parse global_conf and save the parser - global_conf = kwargs.get('global_conf', None) - global_conf_parser = configparser.ConfigParser() - self.config_file = None - self.global_conf_parser = global_conf_parser - if global_conf and "__file__" in global_conf and ".yml" not in global_conf["__file__"]: - self.config_file = global_conf['__file__'] - global_conf_parser.read(global_conf['__file__']) # Heartbeat log file name override - if global_conf is not None and 'heartbeat_log' in global_conf: - self.heartbeat_log = global_conf['heartbeat_log'] + if self.global_conf is not None and 'heartbeat_log' in self.global_conf: + self.heartbeat_log = self.global_conf['heartbeat_log'] if self.heartbeat_log is None: self.heartbeat_log = 'heartbeat_{server_name}.log' # Determine which 'server:' this is @@ -605,7 +628,7 @@ class Configuration(object): self.config_dict['base_server_name'] = self.base_server_name = self.server_name # Store all configured server names for the message queue routing self.server_names = [] - for section in global_conf_parser.sections(): + for section in self.global_conf_parser.sections(): if section.startswith('server:'): self.server_names.append(section.replace('server:', '', 1)) @@ -620,7 +643,7 @@ class Configuration(object): # so dependending on the context a better default can be used ( # request url in a web thread, Docker parent in IE stuff, etc...) galaxy_infrastructure_url = "http://localhost" - web_port = self.galaxy_infrastructure_web_port or self.guess_galaxy_port() + web_port = self.galaxy_infrastructure_web_port if web_port: galaxy_infrastructure_url += ":%s" % (web_port) galaxy_infrastructure_url_set = False @@ -649,7 +672,7 @@ class Configuration(object): elif 'database_connection' in kwargs: self.amqp_internal_connection = "sqlalchemy+" + self.database_connection else: - self.amqp_internal_connection = "sqlalchemy+sqlite:///%s?isolation_level=IMMEDIATE" % resolve_path("database/control.sqlite", self.root) + self.amqp_internal_connection = "sqlalchemy+sqlite:///%s?isolation_level=IMMEDIATE" % resolve_path("control.sqlite", self.data_dir) self.pretty_datetime_format = expand_pretty_datetime_format(kwargs.get('pretty_datetime_format', '$locale (UTC)')) self.user_preferences_extra_config_file = kwargs.get('user_preferences_extra_conf_path', 'config/user_preferences_extra_conf.yml') try: @@ -697,7 +720,7 @@ class Configuration(object): self.gie_swarm_mode = string_as_bool(kwargs.get('interactive_environment_swarm_mode', False)) - self.proxy_session_map = self.resolve_path(kwargs.get("dynamic_proxy_session_map", "database/session_map.sqlite")) + self.proxy_session_map = resolve_path(kwargs.get("dynamic_proxy_session_map", "session_map.sqlite"), self.data_dir) self.manage_dynamic_proxy = string_as_bool(kwargs.get("dynamic_proxy_manage", "True")) # Set to false if being launched externally self.dynamic_proxy_debug = string_as_bool(kwargs.get("dynamic_proxy_debug", "False")) self.dynamic_proxy_bind_port = int(kwargs.get("dynamic_proxy_bind_port", "8800")) @@ -717,8 +740,8 @@ class Configuration(object): self.display_chunk_size = int(kwargs.get('display_chunk_size', 65536)) self.citation_cache_type = kwargs.get("citation_cache_type", "file") - self.citation_cache_data_dir = self.resolve_path(kwargs.get("citation_cache_data_dir", "database/citations/data")) - self.citation_cache_lock_dir = self.resolve_path(kwargs.get("citation_cache_lock_dir", "database/citations/locks")) + self.citation_cache_data_dir = resolve_path(kwargs.get("citation_cache_data_dir", "citations/data"), self.data_dir) + self.citation_cache_lock_dir = resolve_path(kwargs.get("citation_cache_lock_dir", "citations/locks"), self.data_dir) self.containers_conf = parse_containers_config(self.containers_config_file) @@ -790,6 +813,76 @@ class Configuration(object): else: return None + def parse_config_file_options(self, kwargs): + """ + Backwards compatibility for config files moved to the config/ dir. + """ + defaults = dict( + auth_config_file=[self._in_config_dir('auth_conf.xml')], + build_sites_config_file=[self._in_config_dir('build_sites.yml')], + containers_config_file=[self._in_config_dir('containers_conf.yml')], + data_manager_config_file=[self._in_config_dir('data_manager_conf.xml')], + datatypes_config_file=[self._in_config_dir('datatypes_conf.xml'), self._in_sample_dir('datatypes_conf.xml.sample')], + dependency_resolvers_config_file=[self._in_config_dir('dependency_resolvers_conf.xml')], + error_report_file=[self._in_config_dir('error_report.yml')], + job_config_file=[self._in_config_dir('job_conf.xml')], + job_metrics_config_file=[self._in_config_dir('job_metrics_conf.xml')], + job_resource_params_file=[self._in_config_dir('job_resource_params_conf.xml')], + local_conda_mapping_file=[self._in_config_dir('local_conda_mapping.yml')], + migrated_tools_config=[self._in_config_dir('migrated_tools_conf.xml')], + modules_mapping_files=[self._in_config_dir('environment_modules_mapping.yml')], + object_store_config_file=[self._in_config_dir('object_store_conf.xml')], + oidc_backends_config_file=[self._in_config_dir('config/oidc_backends_config.yml')], + oidc_config_file=[self._in_config_dir('oidc_config.yml')], + shed_data_manager_config_file=[self._in_mutable_config_dir('shed_data_manager_conf.xml')], + shed_tool_data_table_config=[self._in_mutable_config_dir('shed_tool_data_table_conf.xml')], + tool_destinations_config_file=[self._in_config_dir('config/tool_destinations.yml')], + tool_sheds_config_file=[self._in_config_dir('tool_sheds_conf.xml')], + workflow_resource_params_file=[self._in_config_dir('workflow_resource_params_conf.xml')], + workflow_schedulers_config_file=[self._in_config_dir('config/workflow_schedulers_conf.xml')], + ) + if running_from_source: + listify_defaults = { + 'tool_data_table_config_path': ['config/tool_data_table_conf.xml', + 'tool_data_table_conf.xml', + 'lib/galaxy/config/sample/tool_data_table_conf.xml.sample'], + # rationale: + # [0]: user has explicitly created config/tool_conf.xml but did not + # move their existing shed_tool_conf.xml, don't use + # config/shed_tool_conf.xml, which is probably the empty + # version copied from the sample, or else their shed tools + # will disappear + # [1]: user has created config/tool_conf.xml and, having passed + # [0], probably moved their shed_tool_conf.xml as well + # [2]: user has done nothing, use the old files + # [3]: fresh install (shed_tool_conf will be added later) + 'tool_config_file': ['config/tool_conf.xml,shed_tool_conf.xml', + 'config/tool_conf.xml,config/shed_tool_conf.xml', + 'tool_conf.xml,shed_tool_conf.xml', + 'lib/galaxy/config/sample/tool_conf.xml.sample'] + } + else: + listify_defaults = { + 'tool_data_table_config_path': [ + self._in_config_dir('tool_data_table_conf.xml'), + self._in_sample_dir('tool_data_table_conf.xml.sample')], + 'tool_config_file': [ + self._in_config_dir('tool_conf.xml'), + self._in_sample_dir('tool_conf.xml.sample')] + } + + self._parse_config_file_options(defaults, listify_defaults, kwargs) + + # If the user has configured a shed tool config in tool_config_file + # this would add a second, but since we're not parsing them yet we + # don't know if that's the case. + if not running_from_source and self.shed_tool_conf not in self.tool_config_file: + self.tool_config_file.append(self.shed_tool_conf) + + # Backwards compatibility for names used in too many places to fix + self.datatypes_config = self.datatypes_config_file + self.tool_configs = self.tool_config_file + def reload_sanitize_whitelist(self, explicit=True): self.sanitize_whitelist = [] try: @@ -801,34 +894,6 @@ class Configuration(object): if explicit: log.warning("Sanitize log file explicitly specified as '%s' but does not exist, continuing with no tools whitelisted.", self.sanitize_whitelist_file) - def __parse_config_file_options(self, kwargs): - """ - Backwards compatibility for config files moved to the config/ dir. - """ - - for var in PATH_DEFAULTS: - setattr(self, var, find_path(kwargs, var, self.root)) - - for var, defaults in PATH_LIST_DEFAULTS.items(): - paths = [] - if kwargs.get(var, None) is not None: - paths = listify(kwargs.get(var)) - else: - for default in defaults: - for path in listify(default): - if not os.path.exists(resolve_path(path, self.root)): - break - else: - paths = listify(default) - break - else: - paths = listify(defaults[-1]) - setattr(self, var, [resolve_path(x, self.root) for x in paths]) - - # Backwards compatibility for names used in too many places to fix - self.datatypes_config = self.datatypes_config_file - self.tool_configs = self.tool_config_file - def get(self, key, default): return self.config_dict.get(key, default) @@ -849,7 +914,7 @@ class Configuration(object): raise ConfigurationError("Unable to create missing directory: %s\n%s" % (path, unicodify(e))) def check(self): - paths_to_check = [self.root, self.tool_path, self.tool_data_path, self.template_path] + paths_to_check = [self.tool_data_path, self.data_dir, self.mutable_config_dir] # Check that required directories exist for path in paths_to_check: if path not in [None, False] and not os.path.isdir(path): @@ -864,10 +929,10 @@ class Configuration(object): self._ensure_directory(path) # Check that required files exist tool_configs = self.tool_configs - if self.migrated_tools_config not in tool_configs: + if self.migrated_tools_config not in tool_configs and os.path.exists(self.migrated_tools_config): tool_configs.append(self.migrated_tools_config) for path in tool_configs: - if not os.path.exists(path): + if not os.path.exists(path) and path != self.shed_tool_conf: raise ConfigurationError("Tool config file not found: %s" % path) for datatypes_config in listify(self.datatypes_config): if not os.path.isfile(datatypes_config): @@ -892,19 +957,6 @@ class Configuration(object): """ return resolve_path(path, self.root) - def guess_galaxy_port(self): - # Code derived from Jupyter work ie.mako - config = configparser.SafeConfigParser({'port': '8080'}) - if self.config_file: - config.read(self.config_file) - - try: - port = config.getint('server:%s' % self.server_name, 'port') - except Exception: - # uWSGI galaxy installations don't use paster and only speak uWSGI not http - port = None - return port - @staticmethod def _parse_allowed_origin_hostnames(kwargs): """ @@ -925,6 +977,10 @@ class Configuration(object): return [parse(v) for v in allowed_origin_hostnames if v] +# legacy naming +Configuration = GalaxyAppConfiguration + + def get_database_engine_options(kwargs, model_prefix=''): """ Allow options for the SQLAlchemy database engine to be passed by using @@ -954,16 +1010,12 @@ def get_database_engine_options(kwargs, model_prefix=''): def get_database_url(config): - if config.database_connection: - db_url = config.database_connection - else: - db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % config.database + db_url = config.database_connection return db_url def init_models_from_config(config, map_install_models=False, object_store=None, trace_logger=None): db_url = get_database_url(config) - from galaxy.model import mapping model = mapping.init( config.file_path, db_url, @@ -1092,9 +1144,14 @@ class ConfiguresGalaxyMixin(object): self.tool_data_tables = ToolDataTableManager(tool_data_path=self.config.tool_data_path, config_filename=self.config.tool_data_table_config_path) # Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables. - self.tool_data_tables.load_from_config_file(config_filename=self.config.shed_tool_data_table_config, - tool_data_path=self.tool_data_tables.tool_data_path, - from_shed_config=from_shed_config) + try: + self.tool_data_tables.load_from_config_file(config_filename=self.config.shed_tool_data_table_config, + tool_data_path=self.tool_data_tables.tool_data_path, + from_shed_config=from_shed_config) + except (OSError, IOError) as exc: + # Missing shed_tool_data_table_config is okay if it's the default + if exc.errno != errno.ENOENT or self.config.shed_tool_data_table_config_set: + raise def _configure_datatypes_registry(self, installed_repository_manager=None): from galaxy.datatypes import registry @@ -1131,7 +1188,7 @@ class ConfiguresGalaxyMixin(object): if os.path.isfile(self.config.tool_sheds_config_file): self.tool_shed_registry = tool_shed.tool_shed_registry.Registry(self.config.tool_sheds_config_file) else: - self.tool_shed_registry = None + self.tool_shed_registry = tool_shed.tool_shed_registry.Registry() def _configure_models(self, check_migrate_databases=False, check_migrate_tools=False, config_file=None): """ @@ -1143,6 +1200,7 @@ class ConfiguresGalaxyMixin(object): # database file under the hood. combined_install_database = not(install_db_url and install_db_url != db_url) install_db_url = install_db_url or db_url + install_database_options = self.config.database_engine_options if combined_install_database else self.config.install_database_engine_options if self.config.database_wait: self._wait_for_database(db_url) @@ -1155,18 +1213,13 @@ class ConfiguresGalaxyMixin(object): # Initialize database / check for appropriate schema version. # If this # is a new installation, we'll restrict the tool migration messaging. from galaxy.model.migrate.check import create_or_verify_database - create_or_verify_database(db_url, config_file, self.config.database_engine_options, app=self) + create_or_verify_database(db_url, config_file, self.config.database_engine_options, app=self, map_install_models=combined_install_database) if not combined_install_database: - from galaxy.model.tool_shed_install.migrate.check import create_or_verify_database as tsi_create_or_verify_database - tsi_create_or_verify_database(install_db_url, self.config.install_database_engine_options, app=self) + tsi_create_or_verify_database(install_db_url, install_database_options, app=self) if check_migrate_tools: # Alert the Galaxy admin to tools that have been moved from the distribution to the tool shed. from tool_shed.galaxy_install.migrate.check import verify_tools - if combined_install_database: - install_database_options = self.config.database_engine_options - else: - install_database_options = self.config.install_database_engine_options verify_tools(self, install_db_url, config_file, install_database_options) self.model = init_models_from_config( @@ -1182,9 +1235,8 @@ class ConfiguresGalaxyMixin(object): from galaxy.model.tool_shed_install import mapping as install_mapping install_db_url = self.config.install_database_connection log.info("Install database using its own connection %s" % install_db_url) - install_db_engine_options = self.config.install_database_engine_options self.install_model = install_mapping.init(install_db_url, - install_db_engine_options) + install_database_options) def _configure_signal_handlers(self, handlers): for sig, handler in handlers.items(): diff --git a/lib/galaxy/config/sample/auth_conf.xml.sample b/lib/galaxy/config/sample/auth_conf.xml.sample new file mode 100644 index 00000000000..34d30693b72 --- /dev/null +++ b/lib/galaxy/config/sample/auth_conf.xml.sample @@ -0,0 +1,128 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + localdb + + + true + + + diff --git a/lib/galaxy/config/sample/build_sites.yml.sample b/lib/galaxy/config/sample/build_sites.yml.sample new file mode 100644 index 00000000000..24a3bed8488 --- /dev/null +++ b/lib/galaxy/config/sample/build_sites.yml.sample @@ -0,0 +1,22 @@ +# Build sites define the builds (dbkeys) available at sites used by display +# applications and the URL to those sites. + +# The `display` attributes on the `ucsc` and `gbrowse` sites replace the +# `ucsc_display_sites` and `gbrowse_display_sites` options in galaxy.ini. +# Because these are used by "old-style" display applications, their types +# cannot change if you want the old-style display links for these sites to +# work. +- type: ucsc + file: tool-data/shared/ucsc/ucsc_build_sites.txt + display: [main,test,archaea,ucla] +- type: gbrowse + file: tool-data/shared/gbrowse/gbrowse_build_sites.txt + display: [modencode,sgd_yeast,tair,wormbase,wormbase_ws120,wormbase_ws140,wormbase_ws170,wormbase_ws180,wormbase_ws190,wormbase_ws200,wormbase_ws204,wormbase_ws210,wormbase_ws220,wormbase_ws225] +- type: ensembl + file: tool-data/shared/ensembl/ensembl_sites.txt +- type: ensembl_data_url + file: tool-data/shared/ensembl/ensembl_sites_data_URL.txt +- type: igv + file: tool-data/shared/igv/igv_build_sites.txt +- type: rviewer + file: tool-data/shared/rviewer/rviewer_build_sites.txt diff --git a/lib/galaxy/config/sample/container_resolvers_conf.xml.sample b/lib/galaxy/config/sample/container_resolvers_conf.xml.sample new file mode 100644 index 00000000000..039f2f5bfe1 --- /dev/null +++ b/lib/galaxy/config/sample/container_resolvers_conf.xml.sample @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/galaxy/config/sample/containers_conf.yml.sample b/lib/galaxy/config/sample/containers_conf.yml.sample new file mode 100644 index 00000000000..71a874cd6ed --- /dev/null +++ b/lib/galaxy/config/sample/containers_conf.yml.sample @@ -0,0 +1,278 @@ +--- +# Galaxy container interface configuration file +# +# To configure the location of this file, use the `containers_config_file` setting in galaxy.yml. Additionally, the +# containers interface is only used if `enable_beta_containers_interface` is set in galaxy.yml. + +### +### Container Interfaces +### + +# Define container interfaces beneath the top-level `containers` dictionary. By default, a single `_default_` interface +# of type `docker` is defined, equivalent to: + +#containers: +# _default_: +# type: docker + +# The interface name is arbitrary and allows multiple distinct configurations to be defined. The name can be any string, +# but `_default_` is used if a component that uses the containers interface does not specify an interface. Currently +# only Galaxy Interactive Environments use the containers interface, and do not have a way to specify which interface to +# use, so configure under the `_default_` key for now. + +# Additional options can be specified are specific to the container type: + +#containers: +# _default_: +# type: docker +# host: docker.example.org:2376 +# force_tlsverify: yes + +# Keys in `containers` are used to map container consumers to specific # container consumers. If a mapping is not +# configured, the _default_ configuration will be used. An example of multiple container configurations would be: + +#containers: +# _default_: +# type: docker +# ... additional options ... +# example_swarm: +# type: docker_swarm +# ... additional options ... +# + +### +### Container Types and Supported Options +### + +# Command-line equivalent arguments are in [brackets] (if applicable) + +containers: + + # + # Supported options for all container types + # + + _default_: + + # [`--name` (partial)] Prepend this string to the name of containers created + #name_prefix: galaxy_ + + # + # Supported options for `docker` container type + # + + local_docker: + + type: docker + + # [`-H`/`--host] Daemon socket(s) to connect to + #host: null + + # [`--tlsverify`] Use TLS and verify the remote + #force_tlsverify: no + + # [`--cpus`] Number of CPUs (default 0.000) + #cpus: null + + # [`-m`/`--memory`] Memory limit + #memory: null + + # Default image to run if one is not provided to the run method + #image: null + + # + # Supported options for `docker_swarm` container type + # + + swarm: + + type: docker_swarm + + # All of the `docker` interface type arguments are supported. Additionally: + + # [`-H`/`--host] Daemon socket(s) to connect to. This can be a list, which allows failing over to another + # manager when one is down, e.g.: + # host: + # - tcp://swarm1.example.org:2376 + # - tcp://swarm2.example.org:2376 + #host: null + + # [`--reserve-cpu` and `--limit-cpu`] Reserve the given number of CPUs when containers are run to prevent other + # containers from being scheduled on the same node once all of its CPUs are allocated. Additionally, prevent + # container from using more than the given number of CPUs. + #cpus: null + + # [`--reserve-memory` and `--limit-memory`] Reserve the given amount of memory when containers are run to + # prevent other containers from being scheduled on the same node once all of its memory is allocated. + # Additionally, prevent container from using more than the given amount of memory. + #memory: null + + # If set, only nodes whose names begin with this string will be visible to and managed by the swarm manager. + # Note that regardless of whether this is set, the swarm manager will not attempt to control manager nodes + #node_prefix: null + + # Convert image from name[:tag] form to name@digest form when possible, to avoid dependency on the image + # registry (e.g. Docker Hub or wherever the image was pulled from) at service creation time. For details see: + # https://github.com/docker/docker/issues/31427 + #resolve_image_digest: no + + # If a container run request is received that defines volumes to attach, should those volumes be ignored? Swarm + # mode does not support mounting volumes. + #ignore_volumes: no + + # + # For the following `service_create_*_constraint` options, if set, the swarm manager automatically sets a + # corresponding label on nodes it spawns for services with the given constraints. + # + + # [`--constraint node.labels._galaxy_image=={image}`] Automatically create a constraint based on the requested + # image name when new services are created (the image name is the resolved form if `resolve_image_digest` is + # set). This is useful if your nodes do not all have all of the possible images you'll want to run. Note: It's + # currently only possible for nodes to have a single image constraint defined. If your nodes have multiple + # images available, do not use this option. + #service_create_image_constraint: no + + # [`--constraint node.labels._galaxy_cpus=={cpus}`] Automatically create a constraint based on the requested + # number of CPUs (or 1, if `cpus` is unset). This is useful if you are spawning nodes with a mixture of CPU + # counts and want to prevent (for example) 1-cpu services from being scheduled on 2-cpu nodes when 2-cpu jobs + # are waiting. + #service_create_cpus_constraint: no + + # + # Galaxy can manage a Docker swarm using a built-in daemon and callouts to commands to add/remove nodes from + # your swarm. + # + + # Use the swarm manager to manage services on this swarm. Even if your swarm is static, using the manager is + # recommended as the manager will automatically remove services after they have terminated. + #managed: yes + + # Automatically start the swarm manager when new services are created. Useful if you want to run the swarm + # manager but control it separately from Galaxy. + #manager_autostart: yes + + # Configuration dictionary for the swarm manager + manager_conf: + + # + # In order to use the swarm manager for more than just service cleanup, (e.g. its node spawn and destroy + # features), you'll need to configure the `cpus` option in the interface. Otherwise, there are no limits on + # the number of services that will be assigned to a given node, so the swarm manager has no way to determine + # that more nodes should be spawned. + # + # Each service consumes one "slot", and the number of slots available on a node is its number of CPUs + # divided by the value of `cpus`. + # + + # When the swarm manager daemonizes, it writes a pid file so that only one manager will run at a time. This + # is the path to that pid file. {xdg_data_home} will be templated automatically and defaults to + # ~/.local/share as per the XDG specification + #pid_file: '{xdg_data_home}/galaxy_swarm_manager.pid' + + # Program output will be written to the log + #log_file: '{xdg_data_home}/galaxy_swarm_manager.log' + + # Level of log messages (levels are Python logging module level names (case insensitive)) + #log_level: INFO + + # Log message format + #log_format: %(name)s %(levelname)s %(asctime)s %(message)s + + # List of services' environment variables that should be logged when performing periodic service logging. By + # default, none are logged, but `USER_EMAIL` is useful for GIE containers to log the user who is running the + # container. + #log_environment_variables: + # - USER_EMAIL + + # Command to run to spawn new nodes. This command should join the node to the swarm. It is run once per + # unique set of constraints of waiting services. Can include template variables: + # + # - {cpus}: Number of CPUs needed by the requested service(s) or minimum limits + # - {slots}: Number of "slots" needed by the requested service(s) or minimum limits, where slots are CPU + # fractions determined by use of the "cpus" option in the `docker_swarm` section. + # - {image}: Image requested by service + # - {service_ids}: Comma-separated list of service ids with these constraints currently waiting + # - {service_count}: Number of services with these constraints currently waiting + # + # If this command does not block until the node is joined to the swarm, make sure it at least completes that + # step in `spawn_wait_time` once it returns control. How the `spawn_command` exits controls the swarm + # manager's behavior: + # + # - return code: 0, output: space-separated list of spawned nodes: + # Swarm manager will wait for the nodes to appear in the swarm and manage them. You are responsible for ensuring + # the names returned match the name as it will appear in the output of `docker node ls`. + # - return code: 0, output: empty: + # Refusing to allocate nodes and the swarm manager should not attempt to spawn nodes for the given service(s) + # again. This is useful for controlling the maximum number of nodes that will be spawned. + # - return code: 2, output: anything (it will be logged as a message) + # Refusing/failed to allocate nodes but the swarm manager should attempt to spawn nodes for the given + # service(s) again. + # + #spawn_command: /bin/true + + # Command to run to destroy idle nodes. Can include template variables: + # + # - {nodes}: Space-separated list of node names to destroy + # + # This command should block until at least the point at which any nodes being destroyed no longer appear in + # the swarm. It should return the names of nodes destroyed, separated by spaces. + # + #destroy_command: /bin/true + + # Command to run if either of the above commands failed (e.g. to notify an administrator). Can include + # template variables: + # + # - {failed_command}: Command line of the command that failed + # + #command_failure_command: /bin/true + + # Number of times to retry spawn/destroy commands before considering them to have failed, and seconds to + # wait between retries. + #command_retries: 0 + #command_retry_wait: 10 + + # Amount of time to wait for a spawning node to appear in the swarm before considering it failed + #spawn_wait_time: 30 + + # Stop the swarm manager daemon when there are no services or nodes to manage + #terminate_when_idle: yes + + # + # Limits control when the swarm manager will spawn and terminate nodes. + # + + # Number of services that must be waiting to run before attempting to spawn a node. If using constraints, + # then waiting services are grouped by constraint. + #service_wait_count_limit: 0 + + # Number of seconds a service must be waiting before attempting to spawn a node. + #service_wait_time_limit: 5 + + # Number of seconds a node must be idle before terminating it. + #node_idle_limit: 120 + + # Minimum number of worker slots that should be started and active. + #slots_min_limit: 0 + + # Number of spare (unused) worker slots that should be started and active + #slots_min_spare: 0 + + # + # Each set of limits can also be set on a per-constraint basis under the `limits` list. Each member of + # `limits` is a dictionary dictionary with at least the key `constraints`, whose value is a list of + # constraint strings matching constraint strings that jobs will be created with, either by the use of + # `service_create_*_constraint` or manually setting constraints. Other keys match the `node_*` and `slots_*` + # limits above. Any limits unset will default to the "global" limits as set above, or their defaults. + # + # If you are using constraints, you should at least define them in the limits section, even if using all + # "global" values for the limits. This allows the swarm manager to know what node types to keep around if + # you're using `slots_min_limit`. + # + + # Limits section example: + #limits: + # constraints: + # - node.labels._galaxy_image==bgruening/docker-jupyter-notebook:16.01.1 + # - node.labels._galaxy_cpus==1 + # slots_min_limit: 2 + # slots_min_spare: 1 diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample new file mode 100644 index 00000000000..3613a6ee58c --- /dev/null +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -0,0 +1,935 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/galaxy/config/sample/dependency_resolvers_conf.xml.sample b/lib/galaxy/config/sample/dependency_resolvers_conf.xml.sample new file mode 100644 index 00000000000..f111e535818 --- /dev/null +++ b/lib/galaxy/config/sample/dependency_resolvers_conf.xml.sample @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/galaxy/config/sample/disposable_email_blacklist.conf.sample b/lib/galaxy/config/sample/disposable_email_blacklist.conf.sample new file mode 100644 index 00000000000..44759454c9c --- /dev/null +++ b/lib/galaxy/config/sample/disposable_email_blacklist.conf.sample @@ -0,0 +1,9 @@ +If you want to disable registration for users that are using disposable email address +rename this file to disposable_email_blacklist.conf and fill it with the disposable domains +that you want to have blacklisted. Each on its own line without the '@' character as shown below. +Users using emails from these domains will get an error during the registration. + +mailinator.com +sogetthis.com +spamgourmet.com +trashmail.net \ No newline at end of file diff --git a/lib/galaxy/config/sample/environment_modules_mapping.yml.sample b/lib/galaxy/config/sample/environment_modules_mapping.yml.sample new file mode 100644 index 00000000000..8fefec6ad55 --- /dev/null +++ b/lib/galaxy/config/sample/environment_modules_mapping.yml.sample @@ -0,0 +1,47 @@ +# This is an example mapping file for the module (http://modules.sourceforge.net/) Dependency resolver (in YAML format) +# +# The goal of this file is to map tool's requirements to existing modules available on your system +# Of course, if the name of a requirement and the name of a module match perfectly, there is no need to map them together through this mapping file. +# +# This is a sample file so the first thing to do to activate the mapping system is to create a copy of this file called "environment_modules_mapping.yml". +# The module dependency resolver is programmed to search and use this YAML file automatically if it exists in the "config" folder of your Galaxy instance. +# Alternatively, you can also use the "mapping_files" attribute of the resolver in the dependency_resolvers_conf.xml file to specify a custom mapping file +# +# Example 1: +# +# Let's say that one of the wrapper installed on your Galaxy instance has the following requirement: +# +# +# PIPITS +# +# +# But unfortunately, the name of the corresponding module file on your system is "pipits_pipeline/1.5.0" +# +# Then, to make Galaxy load/unload the appropriate module, you just have to add the following lines (without the #) to the "environment_modules_mapping.yml" file: +# +#- from: +# name: PIPITS +# version: 1.5.0 +# to: +# name: pipits_pipeline +# version: 1.5.0.6 +# +# +# Example 2: +# +# The requirements section specify a requirement on the PIPITS tool but do not ask for a specific version of it: +# +# +# PIPITS +# +# +# Although, there is no version required you may want to force the loading of a version that is known to run well on your system. +# +# In that case you can add the following lines to the "environment_modules_mapping.yml" file: +# +#- from: +# name: PIPITS +# unversioned: true +# to: +# name: pipits_pipeline +# version: 1.4.0 diff --git a/lib/galaxy/config/sample/error_report.yml.sample b/lib/galaxy/config/sample/error_report.yml.sample new file mode 100644 index 00000000000..cd4ab91ef56 --- /dev/null +++ b/lib/galaxy/config/sample/error_report.yml.sample @@ -0,0 +1,83 @@ +# Each element in this file corresponds to a destination for an error +# report. + +# If you supply the parameter ``user_submission``, and set it to True, +# this error plugin will only be activated when the user uses the bug +# report submission interface. Otherwise, it will be triggered without +# any user interaction (e.g. historical behaviour of sentry plugin) + +# If your plugin is ``user_submission=True``, then supplying +# ``verbose=True`` will cause the plugin to display a message to the end +# user. E.g. the email plugin simply states "Your error report has been +# sent", or the JSON plugin informs the user "Wrote error report to ..." + +# The plugins below are listed with their default values of +# verbose/user_submission, but those are not necessary to provide. + +# The default Email bug reporter. By default, the standard +# configuration is taken from your galaxy.ini +- type: email + verbose: true + user_submission: true + +# Example JSON bug reporter implementation. This just writes the bug +# report as a JSON file to a specified directory and serves as an +# example for others to extend. +# - type: json +# verbose: true +# user_submission: true +# directory: /tmp/reports/ + +# Submit error reports to sentry. If a sentry_dsn is configured in your +# galaxy.ini, then Galaxy will submit the job error to Sentry. You may supply a +# separate DSN for tool reports by supplying a ``custom_dsn`` parameter. +- type: sentry + user_submission: false + +# InfluxDB error reporting backend. You will need to `pip install +# influxdb` in the galaxy virtualenv yourself. This sends well tagged +# errors InfluxDB allowing you to notice relationships between tool errors and +# other infrastructure issues. +# - type: influxdb +# # All arguments prefixed with `influxdb_` are per https://influxdb-python.readthedocs.io/en/latest/api-documentation.html#influxdbclient +# influxdb_host: 127.0.0.1 +# influxdb_port: 8086 +# influxdb_database: galaxy +# influxdb_timeout: 2 + +# Github error reporting backend. You will need to `pip install pygithub` in +# the galaxy virtualenv. This will create a new issue if none exists, and +# comment on existing, open issues. The issues are labelled based on tool ID / +# version and include all of the information the normal emailed bug reports +# include. If you use a private Github Enterprise deployment, you can set +# github_base_url='https://...' and github_api_url='https://api.....' as shown below. +# The 'github_default_repo_only' flag restores the previous behaviour. When this is set +# to true it will automatically only submit to the default git repository. +# - type: github +# verbose: false +# user_submission: true +# github_oauth_token: 00000000000 +# github_base_url: https://github.com +# github_api_url: https://api.github.com +# github_default_repo_owner: galaxyproject +# github_default_repo_name: galaxy +# github_default_repo_only: true + +# GitLab error reporting backend. You will need to `pip install python-gitlab` +# in the galaxy virtualenv. This will create a new issue if none exists, and +# comment on existing, open issues. The issues are labelled based on tool ID / +# version and include all of the information the normal emailed bug reports +# include. If you use a private GitLab deployment, you can set +# gitlab_base_url='https://...'. It supports creating an issue on the git +# repository of the tool by querying the ToolShed where the tool comes from +# (if applicable). Set verbose to true if you want the message to be displayed +# to the user. The 'gitlab_default_repo_only' flags ensures all errors are +# submitted to the default repository only. +# - type: gitlab +# verbose: false +# user_submission: true +# gitlab_base_url: https://gitlab.com +# gitlab_private_token: 00000000000 +# gitlab_default_repo_owner: galaxyproject +# gitlab_default_repo_name: galaxy +# gitlab_default_repo_only: true \ No newline at end of file diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample new file mode 100644 index 00000000000..c0485138c37 --- /dev/null +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -0,0 +1,1779 @@ +# Galaxy is configured by default to be usable in a single-user development +# environment. To tune the application for a multi-user production +# environment, see the documentation at: +# +# https://docs.galaxyproject.org/en/master/admin/production.html +# +# Throughout this sample configuration file, except where stated otherwise, +# uncommented values override the default if left unset, whereas commented +# values are set to the default value. Relative paths are relative to the root +# Galaxy directory. +# +# Examples of many of these options are explained in more detail in the Galaxy +# Community Hub. +# +# https://galaxyproject.org/admin/config +# +# Config hackers are encouraged to check there before asking for help. +# +uwsgi: + + # The address and port on which to listen. By default, only listen to + # localhost (galaxy will not be accessible over the network). Use + # ':8080' to listen on all available network interfaces. + http: 127.0.0.1:8080 + + # By default uWSGI allocates a very small buffer (4096 bytes) for the + # headers of each request. If you start receiving "invalid request + # block size" in your logs, it could mean you need a bigger buffer. We + # recommend at least 16384. + buffer-size: 16384 + + # Number of web server (worker) processes to fork after the + # application has loaded. If this is set to greater than 1, thunder- + # lock likely should be enabled below. + processes: 1 + + # Number of threads for each web server process. + threads: 4 + + # Number of threads for serving static content and handling internal + # routing requests. + offload-threads: 2 + + # Mapping to serve style content. + static-map: /static/style=static/style/blue + + # Mapping to serve the remainder of the static content. + static-map: /static=static + + # Mapping to serve the favicon. + static-map: /favicon.ico=static/favicon.ico + + # Enable the master process manager. Disabled by default for maximum + # compatibility with CTRL+C, but should be enabled for use with + # --daemon and/or production deployments. + master: false + + # Path to the application's Python virtual environment. If using Conda + # for Galaxy's framework dependencies (not tools!), do not set this. + virtualenv: .venv + + # Path to the application's Python library. + pythonpath: lib + + # The entry point which returns the web application (e.g. Galaxy, + # Reports, etc.) that you are loading. + module: galaxy.webapps.galaxy.buildapp:uwsgi_app() + + # Mount the web application (e.g. Galaxy, Reports, etc.) at the given + # URL prefix. Cannot be used together with 'module:' above. + #mount: /galaxy=galaxy.webapps.galaxy.buildapp:uwsgi_app() + + # Make uWSGI rewrite PATH_INFO and SCRIPT_NAME according to mount- + # points. Set this to true if a URL prefix is used. + manage-script-name: false + + # It is usually a good idea to set this to ``true`` if processes is + # greater than 1. + thunder-lock: false + + # Cause uWSGI to respect the traditional behavior of dying on SIGTERM + # (its default is to brutally reload workers) + die-on-term: true + + # Cause uWSGI to gracefully reload workers and mules upon receipt of + # SIGINT (its default is to brutally kill workers) + hook-master-start: unix_signal:2 gracefully_kill_them_all + + # Cause uWSGI to gracefully reload workers and mules upon receipt of + # SIGTERM (its default is to brutally kill workers) + hook-master-start: unix_signal:15 gracefully_kill_them_all + + # Feature necessary for proper mule signal handling + py-call-osafterfork: true + + # Ensure application threads will run if `threads` is unset. + enable-threads: true + +galaxy: + + # By default, Galaxy uses a SQLite database at + # 'database/universe.sqlite'. You may use a SQLAlchemy connection + # string to specify an external database instead. This string takes + # many options which are explained in detail in the config file + # documentation. + #database_connection: sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE + + # If the server logs errors about not having enough database pool + # connections, you will want to increase these values, or consider + # running more Galaxy processes. + #database_engine_option_pool_size: 5 + + # If the server logs errors about not having enough database pool + # connections, you will want to increase these values, or consider + # running more Galaxy processes. + #database_engine_option_max_overflow: 10 + + # If using MySQL and the server logs the error "MySQL server has gone + # away", you will want to set this to some positive value (7200 should + # work). + #database_engine_option_pool_recycle: -1 + + # If large database query results are causing memory or response time + # issues in the Galaxy process, leave the result on the server + # instead. This option is only available for PostgreSQL and is highly + # recommended. + #database_engine_option_server_side_cursors: false + + # Log all database transactions, can be useful for debugging and + # performance profiling. Logging is done via Python's 'logging' + # module under the qualname + # 'galaxy.model.orm.logging_connection_proxy' + #database_query_profiling_proxy: false + + # If auto-creating a postgres database on startup - it can be based on + # an existing template database. This will set that. This is probably + # only useful for testing but documentation is included here for + # completeness. + #database_template: '' + + # Slow query logging. Queries slower than the threshold indicated + # below will be logged to debug. A value of '0' is disabled. For + # example, you would set this to .005 to log all queries taking longer + # than 5 milliseconds. + #slow_query_log_threshold: 0 + + # Enables a per request sql debugging option. If this is set to true, + # append ?sql_debug=1 to web request URLs to enable detailed logging + # on the backend of SQL queries generated during that request. This is + # useful for debugging slow endpoints during development. + #enable_per_request_sql_debugging: false + + # By default, Galaxy will use the same database to track user data and + # tool shed install data. There are many situations in which it is + # valuable to separate these - for instance bootstrapping fresh Galaxy + # instances with pretested installs. The following option can be used + # to separate the tool shed install database (all other options listed + # above but prefixed with install_ are also available). + #install_database_connection: sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE + + # Setting the following option to true will cause Galaxy to + # automatically migrate the database forward after updates. This is + # not recommended for production use. + #database_auto_migrate: false + + # Wait for database to become available instead of failing + # immediately. + #database_wait: false + + # Number of attempts before failing if database_wait is enabled. + #database_wait_attempts: 60 + + # Time to sleep between attempts if database_wait is enabled (in + # seconds). + #database_wait_sleep: 1.0 + + # Where dataset files are stored. It must accessible at the same path + # on any cluster nodes that will run Galaxy jobs, unless using Pulsar. + #file_path: database/files + + # Where temporary files are stored. It must accessible at the same + # path on any cluster nodes that will run Galaxy jobs, unless using + # Pulsar. + #new_file_path: database/tmp + + # Tool config files, defines what tools are available in Galaxy. Tools + # can be locally developed or installed from Galaxy tool sheds. + # (config/tool_conf.xml.sample will be used if left unset and + # config/tool_conf.xml does not exist). + #tool_config_file: config/tool_conf.xml,config/shed_tool_conf.xml + + # Enable / disable checking if any tools defined in the above non-shed + # tool_config_files (i.e., tool_conf.xml) have been migrated from the + # Galaxy code distribution to the Tool Shed. This functionality is + # largely untested in modern Galaxy releases and has serious issues + # such as #7273 and the possibility of slowing down Galaxy startup, so + # the default and recommended value is false. + #check_migrate_tools: false + + # Tool config maintained by tool migration scripts. If you use the + # migration scripts to install tools that have been migrated to the + # tool shed upon a new release, they will be added to this tool config + # file. + #migrated_tools_config: config/migrated_tools_conf.xml + + # File that contains the XML section and tool tags from all tool panel + # config files integrated into a single file that defines the tool + # panel layout. This file can be changed by the Galaxy administrator + # to alter the layout of the tool panel. If not present, Galaxy will + # create it. + #integrated_tool_panel_config: integrated_tool_panel.xml + + # Default path to the directory containing the tools defined in + # tool_conf.xml. Other tool config files must include the tool_path as + # an attribute in the tag. + #tool_path: tools + + # Various dependency resolver configuration parameters will have + # defaults set relative to this path, such as the default conda + # prefix, default Galaxy packages path, legacy tool shed dependencies + # path, and the dependency cache directory. Set the string to None to + # explicitly disable tool dependency handling. If this option is set + # to none or an invalid path, installing tools with dependencies from + # the Tool Shed or in Conda will fail. + #tool_dependency_dir: database/dependencies + + # The dependency resolvers config file specifies an ordering and + # options for how Galaxy resolves tool dependencies (requirement tags + # in Tool XML). The default ordering is to the use the Tool Shed for + # tools installed that way, use local Galaxy packages, and then use + # Conda if available. See https://github.com/galaxyproject/galaxy/blob + # /dev/doc/source/admin/dependency_resolvers.rst for more information + # on these options. + #dependency_resolvers_config_file: config/dependency_resolvers_conf.xml + + # conda_prefix is the location on the filesystem where Conda packages + # and environments are installed IMPORTANT: Due to a current + # limitation in conda, the total length of the conda_prefix and the + # job_working_directory path should be less than 50 characters! + #conda_prefix: /_conda + + # Override the Conda executable to use, it will default to the one on + # the PATH (if available) and then to /bin/conda + #conda_exec: null + + # Pass debug flag to conda commands. + #conda_debug: false + + # conda channels to enable by default (https://conda.io/docs/user- + # guide/tasks/manage-channels.html) + #conda_ensure_channels: iuc,conda-forge,bioconda,defaults + + # Use locally-built conda packages. + #conda_use_local: false + + # Set to true to instruct Galaxy to look for and install missing tool + # dependencies before each job runs. + #conda_auto_install: false + + # Set to true to instruct Galaxy to install Conda from the web + # automatically if it cannot find a local copy and conda_exec is not + # configured. + #conda_auto_init: true + + # You must set this to true if conda_prefix and job_working_directory + # are not on the same volume, or some conda dependencies will fail to + # execute at job runtime. Conda will copy packages content instead of + # creating hardlinks or symlinks. This will prevent problems with some + # specific packages (perl, R), at the cost of extra disk space usage + # and extra time spent copying packages. + #conda_copy_dependencies: false + + # Certain dependency resolvers (namely Conda) take a considerable + # amount of time to build an isolated job environment in the + # job_working_directory if the job working directory is on a network + # share. Set this option to true to cache the dependencies in a + # folder. This option is beta and should only be used if you + # experience long waiting times before a job is actually submitted to + # your cluster. This only affects tools where some requirements can + # be resolved but not others, most modern best practice tools can use + # prebuilt environments in the Conda directory. + #use_cached_dependency_manager: false + + # By default the tool_dependency_cache_dir is the _cache directory of + # the tool dependency directory + #tool_dependency_cache_dir: /_cache + + # By default, when using a cached dependency manager, the dependencies + # are cached when installing new tools and when using tools for the + # first time. Set this to false if you prefer dependencies to be + # cached only when installing new tools. + #precache_dependencies: true + + # File containing the Galaxy Tool Sheds that should be made available + # to install from in the admin interface (.sample used if default does + # not exist). + #tool_sheds_config_file: config/tool_sheds_conf.xml + + # Set to true to enable monitoring of tools and tool directories + # listed in any tool config file specified in tool_config_file option. + # If changes are found, tools are automatically reloaded. Watchdog ( + # https://pypi.org/project/watchdog/ ) must be installed and available + # to Galaxy to use this option. Other options include 'auto' which + # will attempt to watch tools if the watchdog library is available but + # won't fail to load Galaxy if it is not and 'polling' which will use + # a less efficient monitoring scheme that may work in wider range of + # scenarios than the watchdog default. + #watch_tools: 'false' + + # Set to true to enable monitoring of dynamic job rules. If changes + # are found, rules are automatically reloaded. Takes the same values + # as the 'watch_tools' option. + #watch_job_rules: 'false' + + # As of 18.09, Galaxy defaults to setting up the object store + # configuration for output datasets during the job queue step in job + # handlers. This should generally provide for more robust job + # submission, more configurability, and a better user experience but + # may in some cases slightly slow down the job handler job setup + # process. On the off chance that an admin would like to or need to + # optimize job handlers at the expense of user experience and web + # handling this option will remain for some time by setting this + # option to true. This behavior however should be considered + # deprecated and this option will likely be removed in future versions + # of Galaxy. For more information see + # https://github.com/galaxyproject/galaxy/issues/6513. + #legacy_eager_objectstore_initialization: false + + # Enable Galaxy to fetch Docker containers registered with quay.io + # generated from tool requirements resolved through conda. These + # containers (when available) have been generated using mulled - + # https://github.com/mulled . These containers are highly beta and + # availability will vary by tool. This option will additionally only + # be used for job destinations with Docker enabled. + #enable_beta_mulled_containers: false + + # Container resolvers configuration (beta). Set up a file describing + # container resolvers to use when discovering containers for Galaxy. + # If this is set to None, the default containers loaded is determined + # by enable_beta_mulled_containers. + #containers_resolvers_config_file: '' + + # involucro is a tool used to build Docker or Singularity containers + # for tools from Conda dependencies referenced in tools as + # `requirement`s. The following path is the location of involucro on + # the Galaxy host. This is ignored if the relevant container resolver + # isn't enabled, and will install on demand unless involucro_auto_init + # is set to false. + #involucro_path: database/dependencies/involucro + + # Install involucro as needed to build Docker or Singularity + # containers for tools. Ignored if relevant container resolver is not + # used. + #involucro_auto_init: true + + # Conda channels to use when building Docker or Singularity containers + # using involucro. + #mulled_channels: conda-forge,bioconda + + # Enable automatic polling of relative tool sheds to see if any + # updates are available for installed repositories. Ideally only one + # Galaxy server process should be able to check for repository + # updates. The setting for hours_between_check should be an integer + # between 1 and 24. + #enable_tool_shed_check: false + + # Enable automatic polling of relative tool sheds to see if any + # updates are available for installed repositories. Ideally only one + # Galaxy server process should be able to check for repository + # updates. The setting for hours_between_check should be an integer + # between 1 and 24. + #hours_between_check: 12 + + # Enable use of an in-memory registry with bi-directional + # relationships between repositories (i.e., in addition to lists of + # dependencies for a repository, keep an in-memory registry of + # dependent items for each repository. + #manage_dependency_relationships: false + + # XML config file that contains data table entries for the + # ToolDataTableManager. This file is manually # maintained by the + # Galaxy administrator (.sample used if default does not exist). + #tool_data_table_config_path: config/tool_data_table_conf.xml + + # XML config file that contains additional data table entries for the + # ToolDataTableManager. This file is automatically generated based on + # the current installed tool shed repositories that contain valid + # tool_data_table_conf.xml.sample files. At the time of installation, + # these entries are automatically added to the following file, which + # is parsed and applied to the ToolDataTableManager at server start + # up. + #shed_tool_data_table_config: config/shed_tool_data_table_conf.xml + + # Directory where data used by tools is located. See the samples in + # that directory and the Galaxy Community Hub for help: + # https://galaxyproject.org/admin/data-integration + #tool_data_path: tool-data + + # Directory where Tool Data Table related files will be placed when + # installed from a ToolShed. Defaults to tool_data_path. + #shed_tool_data_path: tool-data + + # Set to true to enable monitoring of the tool_data and + # shed_tool_data_path directories. If changes in tool data table files + # are found, the tool data tables for that data manager are + # automatically reloaded. Watchdog ( + # https://pypi.org/project/watchdog/ ) must be installed and available + # to Galaxy to use this option. Other options include 'auto' which + # will attempt to use the watchdog library if it is available but + # won't fail to load Galaxy if it is not and 'polling' which will use + # a less efficient monitoring scheme that may work in wider range of + # scenarios than the watchdog default. + #watch_tool_data_dir: 'false' + + # File containing old-style genome builds + #builds_file_path: tool-data/shared/ucsc/builds.txt + + # Directory where chrom len files are kept, currently mainly used by + # trackster + #len_file_path: tool-data/shared/ucsc/chrom + + # Datatypes config file(s), defines what data (file) types are + # available in Galaxy (.sample is used if default does not exist). If + # a datatype appears in multiple files, the last definition is used + # (though the first sniffer is used so limit sniffer definitions to + # one file). + #datatypes_config_file: config/datatypes_conf.xml + + # Enable sniffing of compressed datatypes. This can be + # configured/overridden on a per-datatype basis in the + # datatypes_conf.xml file. With this option set to false the + # compressed datatypes will be unpacked before sniffing. + #sniff_compressed_dynamic_datatypes_default: true + + # Disable the 'Auto-detect' option for file uploads + #datatypes_disable_auto: false + + # Visualizations config directory: where to look for individual + # visualization plugins. The path is relative to the Galaxy root dir. + # To use an absolute path begin the path with '/'. This is a comma + # separated list. Defaults to "config/plugins/visualizations". + #visualization_plugins_directory: config/plugins/visualizations + + # Interactive environment plugins root directory: where to look for + # interactive environment plugins. By default none will be loaded. + # Set to config/plugins/interactive_environments to load Galaxy's + # stock plugins. These will require Docker to be configured and have + # security considerations, so proceed with caution. The path is + # relative to the Galaxy root dir. To use an absolute path begin the + # path with '/'. This is a comma separated list. + #interactive_environment_plugins_directory: null + + # To run interactive environment containers in Docker Swarm mode (on + # an existing swarm), set this option to true and set + # `docker_connect_port` in the IE plugin config (ini) file(s) of any + # IE plugins you have enabled and ensure that you are not using any + # `docker run`-specific options in your plugins' `command_inject` + # options (swarm mode services run using `docker service create`, + # which has a different and more limited set of options). This option + # can be overridden on a per-plugin basis by using the `swarm_mode` + # option in the plugin's ini config file. + #interactive_environment_swarm_mode: false + + # Galaxy can run a "swarm manager" service that will monitor + # utilization of the swarm and provision/deprovision worker nodes as + # necessary. The service has its own configuration file. + #swarm_manager_config_file: config/swarm_manager_conf.yml + + # Interactive tour directory: where to store interactive tour + # definition files. Galaxy ships with several basic interface tours + # enabled, though a different directory with custom tours can be + # specified here. The path is relative to the Galaxy root dir. To use + # an absolute path begin the path with '/'. This is a comma separated + # list. + #tour_config_dir: config/plugins/tours + + # Webhooks directory: where to store webhooks - plugins to extend the + # Galaxy UI. By default none will be loaded. Set to + # config/plugins/webhooks/demo to load Galaxy's demo webhooks. To use + # an absolute path begin the path with '/'. This is a comma separated + # list. Add test/functional/webhooks to this list to include the demo + # webhooks used to test the webhook framework. + #webhooks_dir: config/plugins/webhooks + + # Each job is given a unique empty directory as its current working + # directory. This option defines in what parent directory those + # directories will be created. + #job_working_directory: database/jobs_directory + + # If using a cluster, Galaxy will write job scripts and stdout/stderr + # to this directory. + #cluster_files_directory: database/pbs + + # Mako templates are compiled as needed and cached for reuse, this + # directory is used for the cache + #template_cache_path: database/compiled_templates + + # Set to false to disable various checks Galaxy will do to ensure it + # can run job scripts before attempting to execute or submit them. + #check_job_script_integrity: true + + # Number of checks to execute if check_job_script_integrity is + # enabled. + #check_job_script_integrity_count: 35 + + # Time to sleep between checks if check_job_script_integrity is + # enabled (in seconds). + #check_job_script_integrity_sleep: 0.25 + + # Set the default shell used by non-containerized jobs Galaxy-wide. + # This defaults to bash for all jobs and can be overridden at the + # destination level for heterogeneous clusters. conda job resolution + # requires bash or zsh so if this is switched to /bin/sh for instance + # - conda resolution should be disabled. Containerized jobs always use + # /bin/sh - so more maximum portability tool authors should assume + # generated commands run in sh. + #default_job_shell: /bin/bash + + # Citation related caching. Tool citations information maybe fetched + # from external sources such as https://doi.org/ by Galaxy - the + # following parameters can be used to control the caching used to + # store this information. + #citation_cache_type: file + + # Citation related caching. Tool citations information maybe fetched + # from external sources such as https://doi.org/ by Galaxy - the + # following parameters can be used to control the caching used to + # store this information. + #citation_cache_data_dir: database/citations/data + + # Citation related caching. Tool citations information maybe fetched + # from external sources such as https://doi.org/ by Galaxy - the + # following parameters can be used to control the caching used to + # store this information. + #citation_cache_lock_dir: database/citations/lock + + # Configuration file for the object store If this is set and exists, + # it overrides any other objectstore settings. + #object_store_config_file: config/object_store_conf.xml + + # What Dataset attribute is used to reference files in an ObjectStore + # implementation, default is 'id' but can also be set to 'uuid' for + # more de-centralized usage. + #object_store_store_by: id + + # Galaxy sends mail for various things: subscribing users to the + # mailing list if they request it, password resets, reporting dataset + # errors, and sending activation emails. To do this, it needs to send + # mail through an SMTP server, which you may define here (host:port). + # Galaxy will automatically try STARTTLS but will continue upon + # failure. + #smtp_server: '' + + # If your SMTP server requires a username and password, you can + # provide them here (password in cleartext here, but if your server + # supports STARTTLS it will be sent over the network encrypted). + #smtp_username: '' + + # If your SMTP server requires a username and password, you can + # provide them here (password in cleartext here, but if your server + # supports STARTTLS it will be sent over the network encrypted). + #smtp_password: '' + + # If your SMTP server requires SSL from the beginning of the + # connection + #smtp_ssl: false + + # On the user registration form, users may choose to join a mailing + # list. This is the address used to subscribe to the list. Uncomment + # and leave empty if you want to remove this option from the user + # registration form. + #mailing_join_addr: galaxy-announce-join@bx.psu.edu + + # Datasets in an error state include a link to report the error. + # Those reports will be sent to this address. Error reports are + # disabled if no address is set. Also this email is shown as a + # contact to user in case of Galaxy misconfiguration and other events + # user may encounter. + #error_email_to: '' + + # Email address to use in the 'From' field when sending emails for + # account activations, workflow step notifications and password + # resets. We recommend using string in the following format: Galaxy + # Project If not configured, '' will be used. + #email_from: '' + + # URL of the support resource for the galaxy instance. Used in + # activation emails. + #instance_resource_url: https://galaxyproject.org/ + + # E-mail domains blacklist is used for filtering out users that are + # using disposable email address during the registration. If their + # address domain matches any domain in the blacklist, they are refused + # the registration. + #blacklist_file: config/disposable_email_blacklist.conf + + # Registration warning message is used to discourage people from + # registering multiple accounts. Applies mostly for the main Galaxy + # instance. If no message specified the warning box will not be shown. + #registration_warning_message: Please register only one account - we provide this service free of charge and have limited computational resources. Multi-accounts are tracked and will be subjected to account termination and data deletion. + + # User account activation feature global flag. If set to false, the + # rest of the Account activation configuration is ignored and user + # activation is disabled (i.e. accounts are active since + # registration). The activation is also not working in case the SMTP + # server is not defined. + #user_activation_on: false + + # Activation grace period (in hours). Activation is not forced (login + # is not disabled) until grace period has passed. Users under grace + # period can't run jobs. Enter 0 to disable grace period. + #activation_grace_period: 3 + + # Shown in warning box to users that were not activated yet. In use + # only if activation_grace_period is set. + #inactivity_box_content: Your account has not been activated yet. Feel free to browse around and see what's available, but you won't be able to upload data or run jobs until you have verified your email address. + + # Password expiration period (in days). Users are required to change + # their password every x days. Users will be redirected to the change + # password screen when they log in after their password expires. Enter + # 0 to disable password expiration. + #password_expiration_period: 0 + + # Galaxy Session Timeout This provides a timeout (in minutes) after + # which a user will have to log back in. A duration of 0 disables this + # feature. + #session_duration: 0 + + # You can enter tracking code here to track visitor's behavior through + # your Google Analytics account. Example: UA-XXXXXXXX-Y + #ga_code: '' + + # Galaxy can display data at various external browsers. These options + # specify which browsers should be available. URLs and builds + # available at these browsers are defined in the specified files. If + # use_remote_user is set to true, display application servers will be + # denied access to Galaxy and so displaying datasets in these sites + # will fail. display_servers contains a list of hostnames which should + # be allowed to bypass security to display datasets. Please be aware + # that there are security implications if this is allowed. More + # details (including required changes to the proxy server config) are + # available in the Apache proxy documentation on the Galaxy Community + # Hub. The list of servers in this sample config are for the UCSC + # Main, Test and Archaea browsers, but the default if left commented + # is to not allow any display sites to bypass security (you must + # uncomment the line below to allow them). + #display_servers: hgw1.cse.ucsc.edu,hgw2.cse.ucsc.edu,hgw3.cse.ucsc.edu,hgw4.cse.ucsc.edu,hgw5.cse.ucsc.edu,hgw6.cse.ucsc.edu,hgw7.cse.ucsc.edu,hgw8.cse.ucsc.edu,lowepub.cse.ucsc.edu + + # Set this to false to disable the old-style display applications that + # are hardcoded into datatype classes. This may be desirable due to + # using the new-style, XML-defined, display applications that have + # been defined for many of the datatypes that have the old-style. + # There is also a potential security concern with the old-style + # applications, where a malicious party could provide a link that + # appears to reference the Galaxy server, but contains a redirect to a + # third-party server, tricking a Galaxy user to access said site. + #enable_old_display_applications: true + + # Show a message box under the masthead. + #message_box_visible: false + + # Show a message box under the masthead. + #message_box_content: '' + + # Class of the message box under the masthead. Possible values are: + # 'info' (the default), 'warning', 'error', 'done'. + #message_box_class: info + + # Append "/{brand}" to the "Galaxy" text in the masthead. + #brand: '' + + # Format string used when showing date and time information. The + # string may contain: - the directives used by Python time.strftime() + # function (see + # https://docs.python.org/library/time.html#time.strftime ), - $locale + # (complete format string for the server locale), - $iso8601 (complete + # format string as specified by ISO 8601 international standard). + #pretty_datetime_format: $locale (UTC) + + # Location of the configuration file containing extra user + # preferences. + #user_preferences_extra_conf_path: config/user_preferences_extra_conf.yml + + # Default localization for Galaxy UI. Allowed values are listed at the + # end of client/galaxy/scripts/nls/locale.js. With the default value + # (auto), the locale will be automatically adjusted to the user's + # navigator language. Users can override this settings in their user + # preferences if the localization settings are enabled in + # user_preferences_extra_conf.yml + #default_locale: auto + + # URL (with schema http/https) of the Galaxy instance as accessible + # within your local network - if specified used as a default by pulsar + # file staging and Jupyter Docker container for communicating back + # with Galaxy via the API. If you are attempting to set up GIEs on + # Mac OS X with Docker Desktop for Mac and your Galaxy instance runs + # on port 8080 this should be 'http://host.docker.internal:8080'. For + # more details see https://docs.docker.com/docker-for-mac/networking/ + #galaxy_infrastructure_url: http://localhost:8080 + + # If the above URL cannot be determined ahead of time in dynamic + # environments but the port which should be used to access Galaxy can + # be - this should be set to prevent Galaxy from having to guess. For + # example if Galaxy is sitting behind a proxy with REMOTE_USER enabled + # - infrastructure shouldn't talk to Python processes directly and + # this should be set to 80 or 443, etc... If unset this file will be + # read for a server block defining a port corresponding to the webapp. + #galaxy_infrastructure_web_port: 8080 + + # The URL of the page to display in Galaxy's middle pane when loaded. + # This can be an absolute or relative URL. + #welcome_url: /static/welcome.html + + # The URL linked by the "Galaxy/brand" text. + #logo_url: / + + # The URL linked by the "Galaxy Help" link in the "Help" menu. + #helpsite_url: '' + + # The URL linked by the "Wiki" link in the "Help" menu. + #wiki_url: https://galaxyproject.org/ + + # The URL linked by the "Support" link in the "Help" menu. + #support_url: https://galaxyproject.org/support/ + + # The URL linked by the "How to Cite Galaxy" link in the "Help" menu. + #citation_url: https://galaxyproject.org/citing-galaxy + + # The URL linked by the "Search" link in the "Help" menu. + #search_url: https://galaxyproject.org/search/ + + # The URL linked by the "Mailing Lists" link in the "Help" menu. + #mailing_lists_url: https://galaxyproject.org/mailing-lists + + # The URL linked by the "Videos" link in the "Help" menu. + #screencasts_url: https://vimeo.com/galaxyproject + + # Points to the GenomeSpace UI service which will be used by the + # GenomeSpace importer and exporter tools + #genomespace_ui_url: https://gsui.genomespace.org/jsui/ + + # The URL linked by the "Terms and Conditions" link in the "Help" + # menu, as well as on the user registration and login forms and in the + # activation emails. + #terms_url: '' + + # The URL linked by the "Galaxy Q&A" link in the "Help" menu The + # Galaxy Q&A site is under development; when the site is done, this + # URL will be set and uncommented. + #qa_url: null + + # Serve static content, which must be enabled if you're not serving it + # via a proxy server. These options should be self explanatory and so + # are not documented individually. You can use these paths (or ones + # in the proxy server) to point to your own styles. + #static_enabled: true + + # Serve static content, which must be enabled if you're not serving it + # via a proxy server. These options should be self explanatory and so + # are not documented individually. You can use these paths (or ones + # in the proxy server) to point to your own styles. + #static_cache_time: 360 + + # Serve static content, which must be enabled if you're not serving it + # via a proxy server. These options should be self explanatory and so + # are not documented individually. You can use these paths (or ones + # in the proxy server) to point to your own styles. + #static_dir: static/ + + # Serve static content, which must be enabled if you're not serving it + # via a proxy server. These options should be self explanatory and so + # are not documented individually. You can use these paths (or ones + # in the proxy server) to point to your own styles. + #static_images_dir: static/images + + # Serve static content, which must be enabled if you're not serving it + # via a proxy server. These options should be self explanatory and so + # are not documented individually. You can use these paths (or ones + # in the proxy server) to point to your own styles. + #static_favicon_dir: static/favicon.ico + + # Serve static content, which must be enabled if you're not serving it + # via a proxy server. These options should be self explanatory and so + # are not documented individually. You can use these paths (or ones + # in the proxy server) to point to your own styles. + #static_scripts_dir: static/scripts/ + + # Serve static content, which must be enabled if you're not serving it + # via a proxy server. These options should be self explanatory and so + # are not documented individually. You can use these paths (or ones + # in the proxy server) to point to your own styles. + #static_style_dir: static/june_2007_style/blue + + # Serve static content, which must be enabled if you're not serving it + # via a proxy server. These options should be self explanatory and so + # are not documented individually. You can use these paths (or ones + # in the proxy server) to point to your own styles. + #static_robots_txt: static/robots.txt + + # Incremental Display Options + #display_chunk_size: 65536 + + # For help on configuring the Advanced proxy features, see: + # https://docs.galaxyproject.org/en/master/admin/production.html + # Apache can handle file downloads (Galaxy-to-user) via mod_xsendfile. + # Set this to true to inform Galaxy that mod_xsendfile is enabled + # upstream. + #apache_xsendfile: false + + # The same download handling can be done by nginx using X-Accel- + # Redirect. This should be set to the path defined in the nginx + # config as an internal redirect with access to Galaxy's data files + # (see documentation linked above). + #nginx_x_accel_redirect_base: '' + + # If using compression in the upstream proxy server, use this option + # to disable gzipping of library .tar.gz and .zip archives, since the + # proxy server will do it faster on the fly. + #upstream_gzip: false + + # The following default adds a header to web request responses that + # will cause modern web browsers to not allow Galaxy to be embedded in + # the frames of web applications hosted at other hosts - this can help + # prevent a class of attack called clickjacking + # (https://www.owasp.org/index.php/Clickjacking). If you configure a + # proxy in front of Galaxy - please ensure this header remains intact + # to protect your users. Uncomment and leave empty to not set the `X + # -Frame-Options` header. + #x_frame_options: SAMEORIGIN + + # nginx can also handle file uploads (user-to-Galaxy) via + # nginx_upload_module. Configuration for this is complex and explained + # in detail in the documentation linked above. The upload store is a + # temporary directory in which files uploaded by the upload module + # will be placed. + #nginx_upload_store: '' + + # This value overrides the action set on the file upload form, e.g. + # the web path where the nginx_upload_module has been configured to + # intercept upload requests. + #nginx_upload_path: '' + + # Galaxy can also use nginx_upload_module to receive files staged out + # upon job completion by remote job runners (i.e. Pulsar) that + # initiate staging operations on the remote end. See the Galaxy nginx + # documentation for the corresponding nginx configuration. + #nginx_upload_job_files_store: '' + + # Galaxy can also use nginx_upload_module to receive files staged out + # upon job completion by remote job runners (i.e. Pulsar) that + # initiate staging operations on the remote end. See the Galaxy nginx + # documentation for the corresponding nginx configuration. + #nginx_upload_job_files_path: '' + + # Galaxy can upload user files in chunks without using nginx. Enable + # the chunk uploader by specifying a chunk size larger than 0. The + # chunk size is specified in bytes (default: 100MB). + #chunk_upload_size: 104857600 + + # Have Galaxy manage dynamic proxy component for routing requests to + # other services based on Galaxy's session cookie. It will attempt to + # do this by default though you do need to install node+npm and do an + # npm install from `lib/galaxy/web/proxy/js`. It is generally more + # robust to configure this externally, managing it in the same way + # Galaxy itself is managed. If true, Galaxy will only launch the + # proxy if it is actually going to be used (e.g. for Jupyter). + #dynamic_proxy_manage: true + + # As of 16.04 Galaxy supports multiple proxy types. The original + # NodeJS implementation, alongside a new Golang single-binary-no- + # dependencies version. Valid values are (node, golang) + #dynamic_proxy: node + + # The NodeJS dynamic proxy can use an SQLite database or a JSON file + # for IPC, set that here. + #dynamic_proxy_session_map: database/session_map.sqlite + + # Set the port and IP for the dynamic proxy to bind to, this must + # match the external configuration if dynamic_proxy_manage is set to + # false. + #dynamic_proxy_bind_port: 8800 + + # Set the port and IP for the dynamic proxy to bind to, this must + # match the external configuration if dynamic_proxy_manage is set to + # false. + #dynamic_proxy_bind_ip: 0.0.0.0 + + # Enable verbose debugging of Galaxy-managed dynamic proxy. + #dynamic_proxy_debug: false + + # The dynamic proxy is proxied by an external proxy (e.g. apache + # frontend to nodejs to wrap connections in SSL). + #dynamic_proxy_external_proxy: false + + # Additionally, when the dynamic proxy is proxied by an upstream + # server, you'll want to specify a prefixed URL so both Galaxy and the + # proxy reside under the same path that your cookies are under. This + # will result in a url like https://FQDN/galaxy-prefix/gie_proxy for + # proxying + #dynamic_proxy_prefix: gie_proxy + + # This attribute governs the minimum length of time between + # consecutive HTTP/WS requests through the proxy, before the proxy + # considers a container as being inactive and kills it. + #dynamic_proxy_golang_noaccess: 60 + + # In order to kill containers, the golang proxy has to check at some + # interval for possibly dead containers. This is exposed as a + # configurable parameter, but the default value is probably fine. + #dynamic_proxy_golang_clean_interval: 10 + + # The golang proxy needs to know how to talk to your docker daemon. + # Currently TLS is not supported, that will come in an update. + #dynamic_proxy_golang_docker_address: unix:///var/run/docker.sock + + # The golang proxy uses a RESTful HTTP API for communication with + # Galaxy instead of a JSON or SQLite file for IPC. If you do not + # specify this, it will be set randomly for you. You should set this + # if you are managing the proxy manually. + #dynamic_proxy_golang_api_key: '' + + # If true, Galaxy will attempt to configure a simple root logger if a + # "loggers" section does not appear in this configuration file. + #auto_configure_logging: true + + # Verbosity of console log messages. Acceptable values can be found + # here: https://docs.python.org/library/logging.html#logging-levels A + # custom debug level of "TRACE" is available for even more verbosity. + #log_level: DEBUG + + # Controls where and how the server logs messages. If unset, the + # default is to log all messages to standard output at the level + # defined by the `log_level` configuration option. Configuration is + # described in the documentation at: + # https://docs.galaxyproject.org/en/master/admin/config_logging.html + #logging: null + + # Print database operations to the server log (warning, quite + # verbose!). + #database_engine_option_echo: false + + # Print database pool operations to the server log (warning, quite + # verbose!). + #database_engine_option_echo_pool: false + + # Turn on logging of application events and some user events to the + # database. + #log_events: true + + # Turn on logging of user actions to the database. Actions currently + # logged are grid views, tool searches, and use of "recently" used + # tools menu. The log_events and log_actions functionality will + # eventually be merged. + #log_actions: true + + # Fluentd configuration. Various events can be logged to the fluentd + # instance configured below by enabling fluent_log. + #fluent_log: false + + # Fluentd configuration. Various events can be logged to the fluentd + # instance configured below by enabling fluent_log. + #fluent_host: localhost + + # Fluentd configuration. Various events can be logged to the fluentd + # instance configured below by enabling fluent_log. + #fluent_port: 24224 + + # Sanitize all HTML tool output. By default, all tool output served + # as 'text/html' will be sanitized thoroughly. This can be disabled + # if you have special tools that require unaltered output. WARNING: + # disabling this does make the Galaxy instance susceptible to XSS + # attacks initiated by your users. + #sanitize_all_html: true + + # Whitelist sanitization file. Datasets created by tools listed in + # this file are trusted and will not have their HTML sanitized on + # display. This can be manually edited or manipulated through the + # Admin control panel -- see "Manage Display Whitelist" + #sanitize_whitelist_file: config/sanitize_whitelist.txt + + # By default Galaxy will serve non-HTML tool output that may + # potentially contain browser executable JavaScript content as plain + # text. This will for instance cause SVG datasets to not render + # properly and so may be disabled by setting this option to true. + #serve_xss_vulnerable_mimetypes: false + + # Return a Access-Control-Allow-Origin response header that matches + # the Origin header of the request if that Origin hostname matches one + # of the strings or regular expressions listed here. This is a comma + # separated list of hostname strings or regular expressions beginning + # and ending with /. E.g. + # mysite.com,google.com,usegalaxy.org,/^[\w\.]*example\.com/ See: + # https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS + #allowed_origin_hostnames: '' + + # Set to true to use Jupyter nbconvert to build HTML from Jupyter + # notebooks in Galaxy histories. This process may allow users to + # execute arbitrary code or serve arbitrary HTML. If enabled, Jupyter + # must be available and on Galaxy's PATH, to do this run `pip install + # jinja2 pygments jupyter` in Galaxy's virtualenv. + #trust_jupyter_notebook_conversion: false + + # Debug enables access to various config options useful for + # development and debugging: use_lint, use_profile, use_printdebug and + # use_interactive. It also causes the files used by PBS/SGE + # (submission script, output, and error) to remain on disk after the + # job is complete. + #debug: false + + # Check for WSGI compliance. + #use_lint: false + + # Run the Python profiler on each request. + #use_profile: false + + # Intercept print statements and show them on the returned page. + #use_printdebug: true + + # Enable live debugging in your browser. This should NEVER be enabled + # on a public site. + #use_interactive: false + + # When stopping Galaxy cleanly, how much time to give various + # monitoring/polling threads to finish before giving up on joining + # them. Set to 0 to disable this and terminate without waiting. Among + # others, these threads include the job handler workers, which are + # responsible for preparing/submitting and collecting/finishing jobs, + # and which can cause job errors if not shut down cleanly. If using + # supervisord, consider also increasing the value of `stopwaitsecs`. + # If using job handler mules, consider also setting the `mule-reload- + # mercy` uWSGI option. See the Galaxy Admin Documentation for more. + #monitor_thread_join_timeout: 30 + + # Write thread status periodically to 'heartbeat.log', (careful, uses + # disk space rapidly!). Useful to determine why your processes may be + # consuming a lot of CPU. + #use_heartbeat: false + + # Control the period (in seconds) between dumps. Use -1 to disable. + # Regardless of this setting, if use_heartbeat is enabled, you can + # send a Galaxy process (unless running with uWSGI) SIGUSR1 (`kill + # -USR1`) to force a dump. + #heartbeat_interval: 20 + + # Heartbeat log filename. Can accept the template variables + # {server_name} and {pid} + #heartbeat_log: heartbeat_{server_name}.log + + # Log to Sentry Sentry is an open source logging and error aggregation + # platform. Setting sentry_dsn will enable the Sentry middleware and + # errors will be sent to the indicated sentry instance. This + # connection string is available in your sentry instance under + # -> Settings -> API Keys. + #sentry_dsn: '' + + # Sentry slow request logging. Requests slower than the threshold + # indicated below will be sent as events to the configured Sentry + # server (above, sentry_dsn). A value of '0' is disabled. For + # example, you would set this to .005 to log all queries taking longer + # than 5 milliseconds. + #sentry_sloreq_threshold: 0.0 + + # Log to statsd Statsd is an external statistics aggregator + # (https://github.com/etsy/statsd) Enabling the following options will + # cause galaxy to log request timing and other statistics to the + # configured statsd instance. The statsd_prefix is useful if you are + # running multiple Galaxy instances and want to segment statistics + # between them within the same aggregator. + #statsd_host: null + + # Log to statsd Statsd is an external statistics aggregator + # (https://github.com/etsy/statsd) Enabling the following options will + # cause galaxy to log request timing and other statistics to the + # configured statsd instance. The statsd_prefix is useful if you are + # running multiple Galaxy instances and want to segment statistics + # between them within the same aggregator. + #statsd_port: 8125 + + # Log to statsd Statsd is an external statistics aggregator + # (https://github.com/etsy/statsd) Enabling the following options will + # cause galaxy to log request timing and other statistics to the + # configured statsd instance. The statsd_prefix is useful if you are + # running multiple Galaxy instances and want to segment statistics + # between them within the same aggregator. + #statsd_prefix: galaxy + + # If you are using telegraf to collect these metrics and then sending + # them to InfluxDB, Galaxy can provide more nicely tagged metrics. + # Instead of sending prefix + dot-separated-path, Galaxy will send + # prefix with a tag path set to the page url + #statsd_influxdb: false + + # Add an option to the library upload form which allows administrators + # to upload a directory of files. + #library_import_dir: '' + + # Add an option to the library upload form which allows authorized + # non-administrators to upload a directory of files. The configured + # directory must contain sub-directories named the same as the non- + # admin user's Galaxy login ( email ). The non-admin user is + # restricted to uploading files or sub-directories of files contained + # in their directory. + #user_library_import_dir: '' + + # If user_library_import_dir is set, this option will auto create a + # library import directory for every user (based on their email) upon + # login. + #user_library_import_dir_auto_creation: false + + # For security reasons, users may not import any files that actually + # lie outside of their `user_library_import_dir` (e.g. using symbolic + # links). A list of directories can be allowed by setting the + # following option (the list is comma-separated). Be aware that *any* + # user with library import permissions can import from anywhere in + # these directories (assuming they are able to create symlinks to + # them). + #user_library_import_symlink_whitelist: '' + + # In conjunction or alternatively, Galaxy can restrict user library + # imports to those files that the user can read (by checking basic + # unix permissions). For this to work, the username has to match the + # username on the filesystem. + #user_library_import_check_permissions: false + + # Allow admins to paste filesystem paths during upload. For libraries + # this adds an option to the admin library upload tool allowing admins + # to paste filesystem paths to files and directories in a box, and + # these paths will be added to a library. For history uploads, this + # allows pasting in paths as URIs. (i.e. prefixed with file://). Set + # to true to enable. Please note the security implication that this + # will give Galaxy Admins access to anything your Galaxy user has + # access to. + #allow_path_paste: false + + # Users may choose to download multiple files from a library in an + # archive. By default, Galaxy allows users to select from a few + # different archive formats if testing shows that Galaxy is able to + # create files using these formats. Specific formats can be disabled + # with this option, separate more than one format with commas. + # Available formats are currently 'zip', 'gz', and 'bz2'. + #disable_library_comptypes: null + + # Some sequencer integration features in beta allow you to + # automatically transfer datasets. This is done using a lightweight + # transfer manager which runs outside of Galaxy (but is spawned by it + # automatically). Galaxy will communicate with this manager over the + # port specified here. + #transfer_manager_port: 8163 + + # Boosts are used to customize this instance's toolbox search. The + # higher the boost, the more importance the scoring algorithm gives to + # the given field. Section refers to the tool group in the tool + # panel. Rest of the fields are tool's attributes. + #tool_name_boost: 9.0 + + # Boosts are used to customize this instance's toolbox search. The + # higher the boost, the more importance the scoring algorithm gives to + # the given field. Section refers to the tool group in the tool + # panel. Rest of the fields are tool's attributes. + #tool_section_boost: 3.0 + + # Boosts are used to customize this instance's toolbox search. The + # higher the boost, the more importance the scoring algorithm gives to + # the given field. Section refers to the tool group in the tool + # panel. Rest of the fields are tool's attributes. + #tool_description_boost: 2.0 + + # Boosts are used to customize this instance's toolbox search. The + # higher the boost, the more importance the scoring algorithm gives to + # the given field. Section refers to the tool group in the tool + # panel. Rest of the fields are tool's attributes. + #tool_label_boost: 1.0 + + # Boosts are used to customize this instance's toolbox search. The + # higher the boost, the more importance the scoring algorithm gives to + # the given field. Section refers to the tool group in the tool + # panel. Rest of the fields are tool's attributes. + #tool_stub_boost: 5.0 + + # Boosts are used to customize this instance's toolbox search. The + # higher the boost, the more importance the scoring algorithm gives to + # the given field. Section refers to the tool group in the tool + # panel. Rest of the fields are tool's attributes. + #tool_help_boost: 0.5 + + # Limits the number of results in toolbox search. Can be used to + # tweak how many results will appear. + #tool_search_limit: 20 + + # Enable/ disable Ngram-search for tools. It makes tool search results + # tolerant for spelling mistakes in the query by dividing the query + # into multiple ngrams and search for each ngram + #tool_enable_ngram_search: false + + # Set minimum size of ngrams + #tool_ngram_minsize: 3 + + # Set maximum size of ngrams + #tool_ngram_maxsize: 4 + + # Set tool test data directory. The test framework sets this value to + # 'test-data,https://github.com/galaxyproject/galaxy-test-data.git' + # which will cause Galaxy to clone down extra test data on the fly for + # certain tools distributed with Galaxy but this is likely not + # appropriate for production systems. Instead one can simply clone + # that repository directly and specify a path here instead of a Git + # HTTP repository. + #tool_test_data_directories: test-data + + # Galaxy encodes various internal values when these values will be + # output in some format (for example, in a URL or cookie). You should + # set a key to be used by the algorithm that encodes and decodes these + # values. It can be any string with a length between 5 and 56 bytes. + # One simple way to generate a value for this is with the shell + # command: python -c 'from __future__ import print_function; import + # time; print(time.time())' | md5sum | cut -f 1 -d ' ' + #id_secret: USING THE DEFAULT IS NOT SECURE! + + # User authentication can be delegated to an upstream proxy server + # (usually Apache). The upstream proxy should set a REMOTE_USER + # header in the request. Enabling remote user disables regular logins. + # For more information, see: https://docs.galaxyproject.org/en/master/ + # admin/special_topics/apache.html + #use_remote_user: false + + # If use_remote_user is enabled and your external authentication + # method just returns bare usernames, set a default mail domain to be + # appended to usernames, to become your Galaxy usernames (email + # addresses). + #remote_user_maildomain: '' + + # If use_remote_user is enabled, the header that the upstream proxy + # provides the remote username in defaults to HTTP_REMOTE_USER (the + # 'HTTP_' is prepended by WSGI). This option allows you to change the + # header. Note, you still need to prepend 'HTTP_' to the header in + # this option, but your proxy server should *not* include 'HTTP_' at + # the beginning of the header name. + #remote_user_header: HTTP_REMOTE_USER + + # If use_remote_user is enabled, anyone who can log in to the Galaxy + # host may impersonate any other user by simply sending the + # appropriate header. Thus a secret shared between the upstream proxy + # server, and Galaxy is required. If anyone other than the Galaxy user + # is using the server, then apache/nginx should pass a value in the + # header 'GX_SECRET' that is identical to the one below. + #remote_user_secret: USING THE DEFAULT IS NOT SECURE! + + # If use_remote_user is enabled, you can set this to a URL that will + # log your users out. + #remote_user_logout_href: '' + + # If your proxy and/or authentication source does not normalize e-mail + # addresses or user names being passed to Galaxy - set this option to + # true to force these to lower case. + #normalize_remote_user_email: false + + # If an e-mail address is specified here, it will hijack remote user + # mechanics (``use_remote_user``) and have the webapp inject a single + # fixed user. This has the effect of turning Galaxy into a single user + # application with no login or external proxy required. Such + # applications should not be exposed to the world. + #single_user: null + + # Administrative users - set this to a comma-separated list of valid + # Galaxy users (email addresses). These users will have access to the + # Admin section of the server, and will have access to create users, + # groups, roles, libraries, and more. For more information, see: + # https://galaxyproject.org/admin/ + #admin_users: '' + + # Force everyone to log in (disable anonymous access). + #require_login: false + + # Show the site's welcome page (see welcome_url) alongside the login + # page (even if require_login is true). + #show_welcome_with_login: false + + # Allow unregistered users to create new accounts (otherwise, they + # will have to be created by an admin). + #allow_user_creation: true + + # Allow administrators to delete accounts. + #allow_user_deletion: false + + # Allow administrators to log in as other users (useful for + # debugging). + #allow_user_impersonation: false + + # When using LDAP for authentication, allow administrators to pre- + # populate users using an additional form on 'Create new user' + #show_user_prepopulate_form: false + + # Allow users to remove their datasets from disk immediately + # (otherwise, datasets will be removed after a time period specified + # by an administrator in the cleanup scripts run via cron) + #allow_user_dataset_purge: true + + # By default, users' data will be public, but setting this to true + # will cause it to be private. Does not affect existing users and + # data, only ones created after this option is set. Users may still + # change their default back to public. + #new_user_dataset_access_role_default_private: false + + # Expose user list. Setting this to true will expose the user list to + # authenticated users. This makes sharing datasets in smaller galaxy + # instances much easier as they can type a name/email and have the + # correct user show up. This makes less sense on large public Galaxy + # instances where that data shouldn't be exposed. For semi-public + # Galaxies, it may make sense to expose just the username and not + # email, or vice versa. If enable_beta_gdpr is set to true, then this + # option will be overridden and set to false. + #expose_user_name: false + + # Expose user list. Setting this to true will expose the user list to + # authenticated users. This makes sharing datasets in smaller galaxy + # instances much easier as they can type a name/email and have the + # correct user show up. This makes less sense on large public Galaxy + # instances where that data shouldn't be exposed. For semi-public + # Galaxies, it may make sense to expose just the username and not + # email, or vice versa. If enable_beta_gdpr is set to true, then this + # option will be overridden and set to false. + #expose_user_email: false + + # Whitelist for local network addresses for "Upload from URL" dialog. + # By default, Galaxy will deny access to the local network address + # space, to prevent users making requests to services which the + # administrator did not intend to expose. Previously, you could + # request any network service that Galaxy might have had access to, + # even if the user could not normally access it. It should be a comma + # separated list of IP addresses or IP address/mask, e.g. + # 10.10.10.10,10.0.1.0/24,fd00::/8 + #fetch_url_whitelist: null + + # Enables GDPR Compliance mode. This makes several changes to the way + # Galaxy logs and exposes data externally such as removing emails and + # usernames from logs and bug reports. It also causes the delete user + # admin action to permanently redact their username and password, but + # not to delete data associated with the account as this is not + # currently easily implementable. You are responsible for removing + # personal data from backups. This forces expose_user_email and + # expose_user_name to be false, and forces user_deletion to be true to + # support the right to erasure. Please read the GDPR section under + # the special topics area of the admin documentation. + #enable_beta_gdpr: false + + # Enable the new container interface for Interactive Environments + #enable_beta_containers_interface: false + + # Enable beta workflow modules that should not yet be considered part + # of Galaxy's stable API. + #enable_beta_workflow_modules: false + + # Default format for the export of workflows. Possible values are 'ga' + # or 'format2'. + #default_workflow_export_format: ga + + # Following options only apply to workflows scheduled using the legacy + # workflow run API (running workflows via a POST to /api/workflows). + # Force usage of Galaxy's beta workflow scheduler under certain + # circumstances - this workflow scheduling forces Galaxy to schedule + # workflows in the background so initial submission of the workflows + # is significantly sped up. This does however force the user to + # refresh their history manually to see newly scheduled steps (for + # "normal" workflows - steps are still scheduled far in advance of + # them being queued and scheduling here doesn't refer to actual + # cluster job scheduling). Workflows containing more than the + # specified number of steps will always use the Galaxy's beta workflow + # scheduling. + #force_beta_workflow_scheduled_min_steps: 250 + + # Following options only apply to workflows scheduled using the legacy + # workflow run API (running workflows via a POST to /api/workflows). + # Force usage of Galaxy's beta workflow scheduler under certain + # circumstances - this workflow scheduling forces Galaxy to schedule + # workflows in the background so initial submission of the workflows + # is significantly sped up. This does however force the user to + # refresh their history manually to see newly scheduled steps (for + # "normal" workflows - steps are still scheduled far in advance of + # them being queued and scheduling here doesn't refer to actual + # cluster job scheduling). Workflows containing more than the + # specified number of steps will always use the Galaxy's beta workflow + # scheduling. Switch to using Galaxy's beta workflow scheduling for + # all workflows involving collections. + #force_beta_workflow_scheduled_for_collections: false + + # If multiple job handlers are enabled, allow Galaxy to schedule + # workflow invocations in multiple handlers simultaneously. This is + # discouraged because it results in a less predictable order of + # workflow datasets within in histories. + #parallelize_workflow_scheduling_within_histories: false + + # This is the maximum amount of time a workflow invocation may stay in + # an active scheduling state in seconds. Set to -1 to disable this + # maximum and allow any workflow invocation to schedule indefinitely. + # The default corresponds to 1 month. + #maximum_workflow_invocation_duration: 2678400 + + # Specify a maximum number of jobs that any given workflow scheduling + # iteration can create. Set this to a positive integer to prevent + # large collection jobs in a workflow from preventing other jobs from + # executing. This may also mitigate memory issues associated with + # scheduling workflows at the expense of increased total DB traffic + # because model objects are expunged from the SQL alchemy session + # between workflow invocation scheduling iterations. Set to -1 to + # disable any such maximum (the default). + #maximum_workflow_jobs_per_scheduling_iteration: -1 + + # Force serial scheduling of workflows within the context of a + # particular history + #history_local_serial_workflow_scheduling: false + + # Enables and disables OpenID Connect (OIDC) support. + #enable_oidc: false + + # Sets the path to OIDC configuration file. + #oidc_config_file: config/oidc_config.xml + + # Sets the path to OIDC backends configuration file. + #oidc_backends_config_file: config/oidc_backends_config.xml + + # XML config file that allows the use of different authentication + # providers (e.g. LDAP) instead or in addition to local authentication + # (.sample is used if default does not exist). + #auth_config_file: config/auth_conf.xml + + # Optional list of email addresses of API users who can make calls on + # behalf of other users. + #api_allow_run_as: '' + + # Master key that allows many API admin actions to be used without + # actually having a defined admin user in the database/config. Only + # set this if you need to bootstrap Galaxy, you probably do not want + # to set this on public servers. + #master_api_key: changethis + + # Enable access to post-authentication options via OpenID. + #enable_openid: false + + # If OpenID is enabled, consumer cache directory to use. + #openid_consumer_cache_path: database/openid_consumer_cache + + # Enable tool tags (associating tools with tags). This has its own + # option since its implementation has a few performance implications + # on startup for large servers. + #enable_tool_tags: false + + # Enable a feature when running workflows. When enabled, default + # datasets are selected for "Set at Runtime" inputs from the history + # such that the same input will not be selected twice, unless there + # are more inputs than compatible datasets in the history. When false, + # the most recently added compatible item in the history will be used + # for each "Set at Runtime" input, independent of others in the + # workflow. + #enable_unique_workflow_defaults: false + + # The URL to the myExperiment instance being used (omit scheme but + # include port). + #myexperiment_url: www.myexperiment.org:80 + + # Enable Galaxy's "Upload via FTP" interface. You'll need to install + # and configure an FTP server (we've used ProFTPd since it can use + # Galaxy's database for authentication) and set the following two + # options. This should point to a directory containing subdirectories + # matching users' identifier (defaults to e-mail), where Galaxy will + # look for files. + #ftp_upload_dir: '' + + # This should be the hostname of your FTP server, which will be + # provided to users in the help text. + #ftp_upload_site: '' + + # User attribute to use as subdirectory in calculating default + # ftp_upload_dir pattern. By default this will be email so a user's + # FTP upload directory will be ${ftp_upload_dir}/${user.email}. Can + # set this to other attributes such as id or username though. + #ftp_upload_dir_identifier: email + + # Python string template used to determine an FTP upload directory for + # a particular user. + #ftp_upload_dir_template: ${ftp_upload_dir}/${ftp_upload_dir_identifier + + # Set to false to prevent Galaxy from deleting uploaded FTP files as + # it imports them. + #ftp_upload_purge: true + + # Enable enforcement of quotas. Quotas can be set from the Admin + # interface. + #enable_quotas: false + + # This option allows users to see the full path of datasets via the + # "View Details" option in the history. This option also exposes the + # command line to non-administrative users. Administrators can always + # see dataset paths. + #expose_dataset_path: false + + # This option allows users to see the job metrics (except for + # environment variables). + #expose_potentially_sensitive_job_metrics: false + + # Enable the API for sample tracking + #enable_legacy_sample_tracking_api: false + + # Allow non-admin users to view available Data Manager options. + #enable_data_manager_user_view: false + + # File where Data Managers are configured (.sample used if default + # does not exist). + #data_manager_config_file: config/data_manager_conf.xml + + # File where Tool Shed based Data Managers are configured. + #shed_data_manager_config_file: config/shed_data_manager_conf.xml + + # Directory to store Data Manager based tool-data; defaults to + # tool_data_path. + #galaxy_data_manager_data_path: tool-data + + # To increase performance of job execution and the web interface, you + # can separate Galaxy into multiple processes. There are more than + # one way to do this, and they are explained in detail in the + # documentation: + # https://docs.galaxyproject.org/en/master/admin/scaling.html By + # default, Galaxy manages and executes jobs from within a single + # process and notifies itself of new jobs via in-memory queues. Jobs + # are run locally on the system on which Galaxy is started. Advanced + # job running capabilities can be configured through the job + # configuration file. + #job_config_file: config/job_conf.xml + + # Description of job running configuration, can be embedded into + # Galaxy configuration or loaded from an additional file with the + # job_config_file option. + #job_config: null + + # Rather than specifying a dependency_resolvers_config_file, the + # definition of the resolvers to enable can be embedded into Galaxy's + # config with this option. This has no effect if a + # dependency_resolvers_config_file is used. + #dependency_resolvers: null + + # Alternative representation of various dependency resolution + # parameters. Takes the dictified version of a DependencyManager + # object - so this is ideal for automating the configuration of + # dependency resolution from one application that uses a + # DependencyManager to another. + #dependency_resolution: null + + # When jobs fail due to job runner problems, Galaxy can be configured + # to retry these or reroute the jobs to new destinations. Very fine + # control of this is available with resubmit declarations in + # job_conf.xml. For simple deployments of Galaxy though, the following + # attribute can define resubmission conditions for all job + # destinations. If any job destination defines even one resubmission + # condition explicitly in job_conf.xml - the condition described by + # this option will not apply to that destination. For instance, the + # condition: 'attempt < 3 and unknown_error and (time_running < 300 or + # time_since_queued < 300)' would retry up to two times jobs that + # didn't fail due to detected memory or walltime limits but did fail + # quickly (either while queueing or running). The commented out + # default below results in no default job resubmission condition, + # failing jobs are just failed outright. + #default_job_resubmission_condition: null + + # This option is deprecated, use the `mem-self` handler assignment + # option in the job configuration instead. + #track_jobs_in_database: true + + # This enables splitting of jobs into tasks, if specified by the + # particular tool config. This is a new feature and not recommended + # for production servers yet. + #use_tasked_jobs: false + + # This enables splitting of jobs into tasks, if specified by the + # particular tool config. This is a new feature and not recommended + # for production servers yet. + #local_task_queue_workers: 2 + + # Enable job recovery (if Galaxy is restarted while cluster jobs are + # running, it can "recover" them when it starts). This is not safe to + # use if you are running more than one Galaxy server using the same + # database. + #enable_job_recovery: true + + # Although it is fairly reliable, setting metadata can occasionally + # fail. In these instances, you can choose to retry setting it + # internally or leave it in a failed state (since retrying internally + # may cause the Galaxy process to be unresponsive). If this option is + # set to false, the user will be given the option to retry externally, + # or set metadata manually (when possible). + #retry_metadata_internally: true + + # Very large metadata values can cause Galaxy crashes. This will + # allow limiting the maximum metadata key size (in bytes used in + # memory, not the end result database value size) Galaxy will attempt + # to save with a dataset. Use 0 to disable this feature. The default + # is 5MB, but as low as 1MB seems to be a reasonable size. + #max_metadata_value_size: 5242880 + + # This option will override tool output paths to write outputs to the + # job working directory (instead of to the file_path) and the job + # manager will move the outputs to their proper place in the dataset + # directory on the Galaxy server after the job completes. This is + # necessary (for example) if jobs run on a cluster and datasets can + # not be created by the user running the jobs (e.g. if the filesystem + # is mounted read-only or the jobs are run by a different user than + # the galaxy user). + #outputs_to_working_directory: false + + # If your network filesystem's caching prevents the Galaxy server from + # seeing the job's stdout and stderr files when it completes, you can + # retry reading these files. The job runner will retry the number of + # times specified below, waiting 1 second between tries. For NFS, you + # may want to try the -noac mount option (Linux) or -actimeo=0 + # (Solaris). + #retry_job_output_collection: 0 + + # In the past Galaxy would preserve its Python environment when + # running jobs ( and still does for internal tools packaged with + # Galaxy). This behavior exposes Galaxy internals to tools and could + # result in problems when activating Python environments for tools + # (such as with Conda packaging). The default legacy_only will + # restrict this behavior to tools identified by the Galaxy team as + # requiring this environment. Set this to "always" to restore the + # previous behavior (and potentially break Conda dependency resolution + # for many tools). Set this to legacy_and_local to preserve the + # environment for legacy tools and locally managed tools (this might + # be useful for instance if you are installing software into Galaxy's + # virtualenv for tool development). + #preserve_python_environment: legacy_only + + # Clean up various bits of jobs left on the filesystem after + # completion. These bits include the job working directory, external + # metadata temporary files, and DRM stdout and stderr files (if using + # a DRM). Possible values are: always, onsuccess, never + #cleanup_job: always + + # When running DRMAA jobs as the Galaxy user + # (https://docs.galaxyproject.org/en/master/admin/cluster.html + # #submitting-jobs-as-the-real-user) this script is used to run the + # job script Galaxy generates for a tool execution. + #drmaa_external_runjob_script: sudo -E scripts/drmaa_external_runner.py --assign_all_groups + + # When running DRMAA jobs as the Galaxy user + # (https://docs.galaxyproject.org/en/master/admin/cluster.html + # #submitting-jobs-as-the-real-user) this script is used to kill such + # jobs by Galaxy (e.g. if the user cancels the job). + #drmaa_external_killjob_script: sudo -E scripts/drmaa_external_killer.py + + # When running DRMAA jobs as the Galaxy user + # (https://docs.galaxyproject.org/en/master/admin/cluster.html + # #submitting-jobs-as-the-real-user) this script is used transfer + # permissions back and forth between the Galaxy user and the user that + # is running the job. + #external_chown_script: sudo -E scripts/external_chown_script.py + + # When running DRMAA jobs as the Galaxy user + # (https://docs.galaxyproject.org/en/master/admin/cluster.html + # #submitting-jobs-as-the-real-user) Galaxy can extract the user name + # from the email address (actually the local-part before the @) or the + # username which are both stored in the Galaxy data base. The latter + # option is particularly useful for installations that get the + # authentication from LDAP. Also, Galaxy can accept the name of a + # common system user (eg. galaxy_worker) who can run every job being + # submitted. This user should not be the same user running the galaxy + # system. Possible values are user_email (default), username or + # + #real_system_username: user_email + + # File to source to set up the environment when running jobs. By + # default, the environment in which the Galaxy server starts is used + # when running jobs locally, and the environment set up per the DRM's + # submission method and policy is used when running jobs on a cluster + # (try testing with `qsub` on the command line). + # environment_setup_file can be set to the path of a file on the + # cluster that should be sourced by the user to set up the environment + # prior to running tools. This can be especially useful for running + # jobs as the actual user, to remove the need to configure each user's + # environment individually. + #environment_setup_file: '' + + # Optional file containing job resource data entry fields definition. + # These fields will be presented to users in the tool forms and allow + # them to overwrite default job resources such as number of + # processors, memory and walltime. + #job_resource_params_file: config/job_resource_params_conf.xml + + # Similar to the above parameter, workflows can describe parameters + # used to influence scheduling of jobs within the workflow. This + # requires both a description of the fields available (which defaults + # to the definitions in job_resource_params_file if not set). + #workflow_resource_params_file: config/workflow_resource_params_conf.xml + + # This parameter describes how to map users and workflows to a set of + # workflow resource parameter to present (typically input IDs from + # workflow_resource_params_file). If this this is a function reference + # it will be passed various inputs (workflow model object and user) + # and it should produce a list of input IDs. If it is a path it is + # expected to an XML or YAML file describing how to map group names to + # parameter descriptions (additional types of mappings via these files + # could be implemented but haven't yet - for instance using workflow + # tags to do the mapping). + #workflow_resource_params_mapper: config/workflow_resource_mapper_conf.yml + + # If using job concurrency limits (configured in job_config_file), + # several extra database queries must be performed to determine the + # number of jobs a user has dispatched to a given destination. By + # default, these queries will happen for every job that is waiting to + # run, but if cache_user_job_count is set to true, it will only happen + # once per iteration of the handler queue. Although better for + # performance due to reduced queries, the trade-off is a greater + # possibility that jobs will be dispatched past the configured limits + # if running many handlers. + #cache_user_job_count: false + + # Define toolbox filters (https://galaxyproject.org/user-defined- + # toolbox-filters/) that admins may use to restrict the tools to + # display. + #tool_filters: null + + # Define toolbox filters (https://galaxyproject.org/user-defined- + # toolbox-filters/) that admins may use to restrict the tool labels to + # display. + #tool_label_filters: null + + # Define toolbox filters (https://galaxyproject.org/user-defined- + # toolbox-filters/) that admins may use to restrict the tool sections + # to display. + #tool_section_filters: null + + # Define toolbox filters (https://galaxyproject.org/user-defined- + # toolbox-filters/) that users may use to restrict the tools to + # display. + #user_tool_filters: examples:restrict_upload_to_admins, examples:restrict_encode + + # Define toolbox filters (https://galaxyproject.org/user-defined- + # toolbox-filters/) that users may use to restrict the tool sections + # to display. + #user_tool_section_filters: examples:restrict_text + + # Define toolbox filters (https://galaxyproject.org/user-defined- + # toolbox-filters/) that users may use to restrict the tool labels to + # display. + #user_tool_label_filters: examples:restrict_upload_to_admins, examples:restrict_encode + + # The base module(s) that are searched for modules for toolbox + # filtering (https://galaxyproject.org/user-defined-toolbox-filters/) + # functions. + #toolbox_filter_base_modules: galaxy.tools.toolbox.filters,galaxy.tools.filters + + # Galaxy uses AMQP internally for communicating between processes. + # For example, when reloading the toolbox or locking job execution, + # the process that handled that particular request will tell all + # others to also reload, lock jobs, etc. For connection examples, see + # http://docs.celeryproject.org/projects/kombu/en/latest/userguide/con + # nections.html Without specifying anything here, galaxy will first + # attempt to use your specified database_connection above. If that's + # not specified either, Galaxy will automatically create and use a + # separate sqlite database located in your /database folder + # (indicated in the commented out line below). + #amqp_internal_connection: sqlalchemy+sqlite:///./database/control.sqlite?isolation_level=IMMEDIATE + + # Galaxy real time communication server settings + #enable_communication_server: false + + # Galaxy real time communication server settings + #communication_server_host: http://localhost + + # Galaxy real time communication server settings + #communication_server_port: 7070 + + # persistent_communication_rooms is a comma-separated list of rooms + # that should be always available. + #persistent_communication_rooms: null + + # Allow disabling pbkdf2 hashing of passwords for legacy situations. + # This should normally be left enabled unless there is a specific + # reason to disable it. + #use_pbkdf2: true + diff --git a/lib/galaxy/config/sample/job_conf.xml.sample_advanced b/lib/galaxy/config/sample/job_conf.xml.sample_advanced new file mode 100644 index 00000000000..471946d0e3a --- /dev/null +++ b/lib/galaxy/config/sample/job_conf.xml.sample_advanced @@ -0,0 +1,1107 @@ + + + + + + + + + + ok + 0 + ok + 0 + + + + /sge/lib/libdrmaa.so + + + + + + + galaxy.jobs.rules + + + + + + + + + + + + amqp://guest:guest@localhost:5672// + + http://localhost:8080 + + + + + + + + + + + + + + + + + + + + + + + + + + + + /path/to/kubeconfig + + + + + + + + + + + galaxy_pvc1:/mount_point1,galaxy_pvc2:/mount_point2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + GODOCKER_URL + + USERNAME + + APIKEY + + galaxy + + + + + `chronos_host` + + foo@bar.com + + username + + password + + true + + + + + + + + + + + + + + + + + + + + + + 4 + + True + + + + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + + + + + + + + + + + + + + + + + + + busybox:ubuntu-14.04 + /path/to/default/container + + + busybox:ubuntu-14.04 + /path/to/default/container + + + + + + walltime=72:00:00 + + + + True + + True + + + + -Xmx6G + '5' + + + + + + + True + + + "$DRM_SET_VARIABLES_FOR_THIS_JOB" + + + $(mktemp -d /mnt/scratch/fastest/gxyjobXXXXXXXXXXX) + + + + + + + foo + + + + dtd + + + choose_one + + cluster1,cluster2,cluster3 + + + + choose_one + cluster1,cluster2,cluster3 + workflow_invocation,history + + + + burst + local_cluster_8_core,local_cluster_1_core,local_cluster_16_core + shared_cluster_8_core + 50 + + + + + burst + local + burst_if_size + 2 + queued + + + python + to_destination_if_size + + galaxycloudrunner.rules + 1g + galaxycloudrunner + local + + + + python + cloudlaunch_pulsar_burst + galaxycloudrunner.rules + https://launch.usegalaxy.org/cloudlaunch/api/v1 + + 37c46c89bcbea797bc7cd76fee10932d2c6a2389 + + pulsar + + local + + + + + + docker_dispatch + docker_cluster + normal_cluster + + + https://examle.com:8913/ + + 123456789changeme + + + + + + + + + + + + + + + + + + + + /path/to/remote/pulsar/files/staging/ + + + + SecureShell + Torque + foo + foo.example.org + walltime=24:00:00,ncpus=4 + + + + + SecureShell + Slurm + foo + my_host + 2:00:00 + 4 + my_partition + + + + + LocalShell + LSF + 16000 + 8 + BigMem + + + + + 8 + + + + + + + + + + + --time=00:05:00 --nodes=1 + + + + + -l h_rt=96:00:00 + + + --mem-per-cpu=512 + + + + + + + + + --time=00:05:00 --nodes=1 + false + + + + + + + my-docker-registry.org + superbioinfo + my-tool + latest + + + 3 + + + true + + + + true + 1 + 2 + centos:latest + + + + false + + + + true + 512 + 2 + /directory/ + + 2 + + + + + + + + + + + + + memory + processors,memory,time,project + + + + + + + + + + + + + + trackster + + + + + + 2 + + 1 + + 1 + 2 + 1 + + 16 + 100 + + 24:00:00 + + 24:00:00 + + 10GB + + + + + SecureShell + Torque + galaxy + foohost_destination.example.org + walltime=@WALLTIME@,ncpus=@NCPUS@ + + + + diff --git a/lib/galaxy/config/sample/job_conf.xml.sample_basic b/lib/galaxy/config/sample/job_conf.xml.sample_basic new file mode 100644 index 00000000000..0f10244f48c --- /dev/null +++ b/lib/galaxy/config/sample/job_conf.xml.sample_basic @@ -0,0 +1,11 @@ + + + + + + + + + + diff --git a/lib/galaxy/config/sample/job_metrics_conf.xml.sample b/lib/galaxy/config/sample/job_metrics_conf.xml.sample new file mode 100644 index 00000000000..1c9c70274e5 --- /dev/null +++ b/lib/galaxy/config/sample/job_metrics_conf.xml.sample @@ -0,0 +1,138 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/galaxy/config/sample/job_resource_params_conf.xml.sample b/lib/galaxy/config/sample/job_resource_params_conf.xml.sample new file mode 100644 index 00000000000..c9adce5cb97 --- /dev/null +++ b/lib/galaxy/config/sample/job_resource_params_conf.xml.sample @@ -0,0 +1,6 @@ + + + + + + diff --git a/lib/galaxy/config/sample/lmod_modules_mapping.yml.sample b/lib/galaxy/config/sample/lmod_modules_mapping.yml.sample new file mode 100644 index 00000000000..f916e2ce539 --- /dev/null +++ b/lib/galaxy/config/sample/lmod_modules_mapping.yml.sample @@ -0,0 +1,47 @@ +# This is an example mapping file for the LMOD Dependency resolver (in YAML format) +# +# The goal of this file is to map tool's requirements to existing LMOD modules available on your system +# Of course, if the name of a requirement and the name of a module match perfectly, there is no need to map them together through this mapping file. +# +# This is a sample file so the first thing to do to activate the mapping system is to create a copy of this file called "lmod_modules_mapping.yml". +# The Lmod dependency resolver is programmed to search and use this YAML file automatically if it exists in the "config" folder of your Galaxy instance. +# Alternatively, you can also use the "mapping_files" attribute of the resolver in the dependency_resolvers_conf.xml file to specify a custom mapping file +# +# Example 1: +# +# Let's say that one of the wrapper installed on your Galaxy instance has the following requirement: +# +# +# PIPITS +# +# +# But unfortunately, the name of the corresponding module file on your system is "pipits_pipeline/1.5.0" +# +# Then, to make Galaxy load/unload the appropriate module, you just have to add the following lines (without to the #) to the "lmod_modules_mapping.yml" file: +# +#- from: +# name: PIPITS +# version: 1.5.0 +# to: +# name: pipits_pipeline +# version: 1.5.0.6 +# +# +# Example 2: +# +# The requirements section specify a requirement on the PIPITS tool but do not ask for a specific version of it: +# +# +# PIPITS +# +# +# Although, there is no version required you may want to force the loading of a version that is known to run well on your system. +# +# In that case you can add the following lines to the "lmod_modules_mapping.yml" file: +# +#- from: +# name: PIPITS +# unversioned: true +# to: +# name: pipits_pipeline +# version: 1.4.0 diff --git a/lib/galaxy/config/sample/local_conda_mapping.yml.sample b/lib/galaxy/config/sample/local_conda_mapping.yml.sample new file mode 100644 index 00000000000..bc9191acbb5 --- /dev/null +++ b/lib/galaxy/config/sample/local_conda_mapping.yml.sample @@ -0,0 +1,2 @@ +# See $GALAXY_ROOT/lib/galaxy/tools/deps/resolvers/default_conda_mapping.yml for example mapping - +# additional site-specific mappings can be added to config/local_conda_mapping.yml. diff --git a/lib/galaxy/config/sample/object_store_conf.xml.sample b/lib/galaxy/config/sample/object_store_conf.xml.sample new file mode 100644 index 00000000000..12dbf9aef5b --- /dev/null +++ b/lib/galaxy/config/sample/object_store_conf.xml.sample @@ -0,0 +1,104 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/galaxy/config/sample/oidc_backends_config.xml.sample b/lib/galaxy/config/sample/oidc_backends_config.xml.sample new file mode 100644 index 00000000000..90b157b1632 --- /dev/null +++ b/lib/galaxy/config/sample/oidc_backends_config.xml.sample @@ -0,0 +1,111 @@ + + + + + ... + ... + http://localhost:8080/authnz/google/callback + + consent + + + + + + ... + ... + http://localhost:8080/authnz/globus/callback + + consent + + + + ... + ... + ... + http://localhost:8000/authnz/custos/callback + + ... + + + + + + + + + + ... + ... + http://localhost:8080/authnz/elixir/callback + consent + + + diff --git a/lib/galaxy/config/sample/oidc_config.xml.sample b/lib/galaxy/config/sample/oidc_config.xml.sample new file mode 100644 index 00000000000..f8618dd74dd --- /dev/null +++ b/lib/galaxy/config/sample/oidc_config.xml.sample @@ -0,0 +1,21 @@ + + + + + + + + + diff --git a/lib/galaxy/config/sample/reports.yml.sample b/lib/galaxy/config/sample/reports.yml.sample new file mode 100644 index 00000000000..5ca6bdd5008 --- /dev/null +++ b/lib/galaxy/config/sample/reports.yml.sample @@ -0,0 +1,131 @@ +uwsgi: + + # The address and port on which to listen. By default, only listen to + # localhost (reports will not be accessible over the network). Use + # ':9001' to listen on all available network interfaces. + http: 127.0.0.1:9001 + + # By default uWSGI allocates a very small buffer (4096 bytes) for the + # headers of each request. If you start receiving "invalid request + # block size" in your logs, it could mean you need a bigger buffer. We + # recommend at least 16384. + buffer-size: 16384 + + # Number of web server (worker) processes to fork after the + # application has loaded. If this is set to greater than 1, thunder- + # lock likely should be enabled below. + processes: 1 + + # Number of threads for each web server process. + threads: 4 + + # Number of threads for serving static content and handling internal + # routing requests. + offload-threads: 2 + + # Mapping to serve style content. + static-map: /static/style=static/style/blue + + # Mapping to serve the remainder of the static content. + static-map: /static=static + + # Mapping to serve the favicon. + static-map: /favicon.ico=static/favicon.ico + + # Enable the master process manager. Disabled by default for maximum + # compatibility with CTRL+C, but should be enabled for use with + # --daemon and/or production deployments. + master: false + + # Path to the application's Python virtual environment. If using Conda + # for Galaxy's framework dependencies (not tools!), do not set this. + virtualenv: .venv + + # Path to the application's Python library. + pythonpath: lib + + # The entry point which returns the web application (e.g. Galaxy, + # Reports, etc.) that you are loading. + module: galaxy.webapps.reports.buildapp:uwsgi_app() + + # Mount the web application (e.g. Galaxy, Reports, etc.) at the given + # URL prefix. Cannot be used together with 'module:' above. + #mount: /galaxy=galaxy.webapps.galaxy.buildapp:uwsgi_app() + + # Make uWSGI rewrite PATH_INFO and SCRIPT_NAME according to mount- + # points. Set this to true if a URL prefix is used. + manage-script-name: false + + # It is usually a good idea to set this to ``true`` if processes is + # greater than 1. + thunder-lock: false + + # Cause uWSGI to respect the traditional behavior of dying on SIGTERM + # (its default is to brutally reload workers) + die-on-term: true + + # Cause uWSGI to gracefully reload workers and mules upon receipt of + # SIGINT (its default is to brutally kill workers) + hook-master-start: unix_signal:2 gracefully_kill_them_all + + # Cause uWSGI to gracefully reload workers and mules upon receipt of + # SIGTERM (its default is to brutally kill workers) + hook-master-start: unix_signal:15 gracefully_kill_them_all + + # Feature necessary for proper mule signal handling + py-call-osafterfork: true + + # Ensure application threads will run if `threads` is unset. + enable-threads: true + +reports: + + # Verbosity of console log messages. Acceptable values can be found + # here: https://docs.python.org/2/library/logging.html#logging-levels + #log_level: DEBUG + + # Database connection. Galaxy Reports are intended for production + # Galaxy instances, so sqlite (and the default value below) is not + # supported. An SQLAlchemy connection string should be used specify an + # external database. + #database_connection: sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE + + # Where dataset files are stored. + #file_path: database/files + + # Where temporary files are stored. + #new_file_path: database/tmp + + # Mako templates are compiled as needed and cached for reuse, this + # directory is used for the cache + #template_cache_path: database/compiled_templates/reports + + # Configuration for debugging middleware + #debug: false + + # Check for WSGI compliance. + #use_lint: false + + # NEVER enable this on a public site (even test or QA) + #use_interactive: false + + # Write thread status periodically to 'heartbeat.log' (careful, uses + # disk space rapidly!) + #use_heartbeat: true + + # Profiling middleware (cProfile based) + #use_profile: true + + # Mail + #smtp_server: yourserver@yourfacility.edu + + # Mail + #error_email_to: your_bugs@bx.psu.edu + + # Enables GDPR Compliance mode. This makes several changes to the way + # Galaxy logs and exposes data externally such as removing + # emails/usernames from logs and bug reports. You are responsible for + # removing personal data from backups. Please read the GDPR section + # under the special topics area of the admin documentation. + #enable_beta_gdpr: false + diff --git a/lib/galaxy/config/sample/swarm_manager_conf.yml.sample b/lib/galaxy/config/sample/swarm_manager_conf.yml.sample new file mode 100644 index 00000000000..31c5632519c --- /dev/null +++ b/lib/galaxy/config/sample/swarm_manager_conf.yml.sample @@ -0,0 +1,84 @@ +--- +# Galaxy docker swarm manager configuration file +# +# To configure the location of this file, use the `swarm_manager_config_file` +# setting in galaxy.ini + +# When the swarm manager daemonizes, it writes a pid file so that only one +# manager will run at a time. This is the path to that pid file. +# {xdg_data_home} will be templated automatically and defaults to +# ~/.local/share as per the XDG specification +#pid_file: '{xdg_data_home}/galaxy_swarm_manager.pid' + +# Program output will be written to the log +#log_file: '{xdg_data_home}/galaxy_swarm_manager.log' + +# As with GIE plugins, you can modify the base docker command ({docker_args} +# must be present and will be filled in with the docker subcommand and +# arguments) +#command: 'docker {docker_args}' + +# Managed services should be started with this string at the beginning of their +# name. It should match the value of CONTAINER_NAME_PREFIX in +# lib/galaxy/web/base/interactive_environments.py, so you should not change +# this unless you change both. +#service_prefix: galaxy_gie_ + +# Limits: +# +# - max_waiting_services: number of services that should be waiting of each +# "CPU class" (number of CPUs requested e.g. with --reserve-cpu) before +# attempting to spawn a node +# - max_wait_time: number of seconds a service should be waiting before +# attempting to spawn a node +# - max_node_idle_time: number of seconds a node should be idle before +# terminating it +# - max_node_counts: a dictionary controlling the maximum number of nodes of +# each CPU class that the swarm manager will attempt to spawn, e.g.: +# max_node_counts: +# 1: 10 # spawn up to 10 x 1-CPU nodes +# 2: 3 # spawn up to 3 x 2-CPU nodes +# 4: 1 # spawn up to 1 x 4-CPU nodes +#max_waiting_services: 0 +#max_wait_time: 5 +#max_node_idle_time: 120 +#max_node_counts: {} + +# If set, only manage nodes whose swarm hostnames begin with this prefix. +# Otherwise, attempt to manage all nodes +#node_prefix: null + +# Amount of time to wait for a spawning node to appear in `docker node ls` +# before considering it failed +#spawn_wait_time: 30 + +# Command to run to spawn new nodes. This command should join the node to the +# swarm. Can include template variables: +# - {cpu_class}: CPU class as explained above (the value of --reserve-cpu) +# - {cpus_needed}: Total number of CPUs of the given class needed to run the +# waiting services +# If this command does not block until the node is joined to the swarm, make +# sure it at least completes that step in `spawn_wait_time` once it returns +# control. This command should return a space-separated list of nodes. If the +# nodes have a different number of CPUs than the class that they were started +# for, you can include that number after a colon (e.g. `node1:4`). +#spawn_command: /bin/true + +# Command to run to destroy idle nodes. Can include template variables: +# - {nodes}: Space-separated list of node names to destroy +# This command should block until at least the point at which any nodes being +# deallocated no longer appear in `docker node ls`. +#destroy_command: /bin/true + +# Command to run if either of the above commands failed (e.g. to notify an +# administrator). Can include template variables: +# - {failed_command}: Command line of the command that failed +#command_failure_command: /bin/true + +# Number of times to retry spawn/destroy commands before considering them to +# have failed, and seconds to wait between retries +#command_retries: 0 +#command_retry_wait: 10 + +# Stop the swarm manager daemon when there are no services or nodes to manage +#terminate_when_idle: True diff --git a/lib/galaxy/config/sample/tool_conf.xml.main b/lib/galaxy/config/sample/tool_conf.xml.main new file mode 100644 index 00000000000..99aa2052798 --- /dev/null +++ b/lib/galaxy/config/sample/tool_conf.xml.main @@ -0,0 +1,142 @@ + + +
+ + + + + + + + + + + + + + + + + +
+
+ + +
+
+ +
+
+ + + + + + + + + +
+
+ +
+
+ + + + + + + + +
+
+ + + +
+
+ + + +
+
+ + + + + + + + + + + + +
+
+ + + +
+
+ + +
+
+
+
+ + + + + + + + + + + +
+
+
+
+
+
diff --git a/lib/galaxy/config/sample/tool_conf.xml.sample b/lib/galaxy/config/sample/tool_conf.xml.sample new file mode 100644 index 00000000000..aeb42d29f3b --- /dev/null +++ b/lib/galaxy/config/sample/tool_conf.xml.sample @@ -0,0 +1,129 @@ + + +
+ + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+
+ + + + + + + + + + + + + +
+
+ +
+
+ +
+
+ + + + + + + + + + + + + + + + +
+
+ + + + + + + + +
+
+ + + +
+
+ + + +
+
+ + + + + + + + + + + + +
+
+ + + +
+
+ + +
+
+ +
+
+ + + + + + + +
+
diff --git a/lib/galaxy/config/sample/tool_data_table_conf.xml.sample b/lib/galaxy/config/sample/tool_data_table_conf.xml.sample new file mode 100644 index 00000000000..a3ef84c653c --- /dev/null +++ b/lib/galaxy/config/sample/tool_data_table_conf.xml.sample @@ -0,0 +1,108 @@ + + + + + value, dbkey, name, path + +
+ + + value, dbkey, formats, name, path + +
+ + + value, name, path + +
+ + + value, name, path + +
+ + + value, name, path + +
+ + + value, dbkey, name, path + +
+ + + value, dbkey, name, path + +
+ + + name, value, dbkey, species + +
+ + + value, dbkey, name, path + +
+ + + value, name, path + +
+ + + value, name, path + +
+ + + value, dbkey, name, path + +
+ + + value, dbkey, name, path + +
+ + + value, dbkey, name, path + +
+ + + value, path + +
+ + + name, url, value + +
+ + + dbkey, name, value + +
+ + + value, name, url + +
+ + + value, name, url + +
+ + + value, name, url + +
+ + + value, name, url + +
+
diff --git a/lib/galaxy/config/sample/tool_destinations.yml.sample b/lib/galaxy/config/sample/tool_destinations.yml.sample new file mode 100644 index 00000000000..9f96b3e8d6f --- /dev/null +++ b/lib/galaxy/config/sample/tool_destinations.yml.sample @@ -0,0 +1,313 @@ +# Dynamic Tool Destinations config has four main levels (or key properties) - tools, +# default_destination, users, and verbose. +# +# Under tools is where you put the IDs of the tools. Tool IDs must be +# distinct. In each tool is a list of rules, each of which contain rule-specific +# parameters. Dynamic Tool Destination is set up to allow and fix a few light errors +# in the config, but it's best to follow the specified template. +# +# default_destination, the second key property, specifies which global destination +# to default to in case none of the rules apply. +# +# users, the third key property, can be used to assign a priority to a user. Three priorities +# are available low, med, and high. By default all users are assumed to be med priority. If +# a rule has priority destinations the user will be set to the destination which matches their priority +# or if their priority isn't available they will be matched to the next closest one. +# +# A fourth key property is verbose. When this is set to True, Dynamic Tool Destination +# gives much more descriptive output regarding the steps it's taking in mapping your tools to +# the appropriate destinations, including config validation and any potential errors found in the +# config. +# +# Note: Configurations can be validated with the following command: +# +# python lib/galaxy/jobs/dynamic_tool_destination.py -c +# +# or +# +# python lib/galaxy/jobs/dynamic_tool_destination.py -c /path/to/tool_destinations.yml +# +# The general template is as follows (note that this template +# does not use quote symbols), using spades and smalt as an example +# (spades for showing what each field is for, and smalt +# to give a fairly real-world example): +# +# Ex: +# +# tools: +# spades: +# rules: +# - rule_type: what kind of rule is it? +# nice_value: what kind of priority does this rule have over others? +# destination: how should this tool be run? +# lower_bound: what's the max file size? +# upper_bound: what's the minimum file size? +# users: #(optional) which users does this rule apply to +# - user@example.com +# default_destination: this tool-specific field is optional +# smalt_map: +# rules: +# - rule_type: file_size +# nice_value: 0 +# lower_bound: 0 +# upper_bound: 2 GB +# destination: cluster_low_4 +# - rule_type: file_size +# nice_value: 0 +# lower_bound: 2 GB +# upper_bound: 4 GB +# destination: cluster_low_8 +# - rule_type: file_size +# nice_value: 0 +# lower_bound: 4 GB +# upper_bound: Infinity +# destination: cluster_low_16 +# default_destination: cluster_default +# users: +# user_low@example.com: +# priority: low +# user_med@example.com: +# priority: med +# user_high@example.com: +# priority: high +# default_destination: this global field is mandatory +# verbose: True +# +# +# +# Looking at this example, some things must be clarified: each entry in the list of +# rules per tool is specified by '-'. Per rule, regardless of rule type, +# the following fields are mandatory: +# rule_type, nice_value, and destination. +# +# Some of the other fields are mandatory only for specific rule types, which will be +# further discussed below. +# +# Starting with rule_type, there are currently 3 rule types: file_size, records, +# and arguments. +# +# file_size and records rules are based on how large the files are: if they fall +# within specified limits, then the rule is satisfied, and the tool may proceed +# with the appropriate destination. +# +# file_size and records rules have the following required parameters on top of the base +# mandatory parameters: +# upper_bound +# lower_bound +# +# Bounds are allowed to be specified in bytes (48000 for example) or a higher size unit, +# including the unit abbreviation (4 GB or 10 TB for example). Additionally, upper_bound +# is allowed to be Infinite; simply specify Infinite in order to do so. +# +# **The rule will allow the lower_bound, up to but not including the upper_bound +# +# The third rule_type is arguments, which has arguments as a mandatory parameter on top of +# the base mandatory parameters. The arguments parameter is specified using the following +# template: +# +# +# arguments: +# argument_name: the_argument +# +# +# A real world example is shown below: +# +# +# tools: +# spades: +# rules: +# - rule_type: arguments +# nice_value: -19 +# destination: fail +# fail_message: Don't do that +# arguments: +# my_section: +# my_condition: +# careful: true +# default_destination: cluster_low +# default_destination: cluster +# verbose: False +# +# +# Next up, nice_value is used for prioritizing rules over others in case two rules +# match. nice_value basically translates to, "the higher the nice_value, the 'nicer' +# the tool is about being picked last". So based off of that idea, a rule with a nice +# value of -5 is guaranteed to be picked over a rule with a nice value of 10. nice_value +# is allowed to go from -20 to 20. If two rules have the same nice value and both were +# satisfied, the first rule in the config file will be picked. In summary, first-come- +# first-serve basis unless nice_value overrides that. +# +# +# Finally, destination simply refers to the specific way the tool will run. Each +# destination ID refers to a specific configuration to run the tool with. +# +# Some rules may call for the job to fail if certain conditions are encountered. In +# this case, destination simply refers to 'fail'. +# +# For example, the following rule is set to fail the job if a file that is too large +# (more than 4GB) is encountered: +# +# +# tools: +# spades: +# rules: +# - rule_type: file_size +# nice_value: 0 +# destination: fail +# fail_message: Data too large +# lower_bound: 4 GB +# upper_bound: Infinity +# +# +# As shown above, a rule with 'fail' as the destination requires an additional +# parameter, 'fail_message', which DynamicToolDestination uses to print a helpful error +# message to the user indicating why the job failed (showing up inside the job log in +# Galaxy's history panel). + + +tools: + spades: + rules: + - rule_type: file_size + nice_value: 0 + lower_bound: 0 + upper_bound: 10 MB + fail_message: Too few reads for spades to work + destination: fail + - rule_type: file_size + nice_value: 0 + lower_bound: 10 MB + upper_bound: 2 GB + destination: + priority: + low: slurm_low_24 + med: slurm_med_24 + high: slurm_high_24 + - rule_type: file_size + nice_value: 0 + lower_bound: 2 GB + upper_bound: 4 GB + destination: + priority: + low: slurm_low_48 + med: slurm_med_48 + high: slurm_high_48 + - rule_type: file_size + nice_value: 0 + lower_bound: 4 GB + upper_bound: Infinity + fail_message: Too much data, shoudn't run + destination: fail + - rule_type: arguments + nice_value: 0 + arguments: + careful: true + destination: + priority: + low: slurm_low_48 + med: slurm_med_48 + high: slurm_high_48 + default_destination: + priority: + low: slurm_low_16 + med: slurm_med_16 + high: slurm_high_16 + + srst2: + rules: + - rule_type: records + nice_value: 0 + lower_bound: 0 + upper_bound: 100 + destination: + priority: + low: slurm_low_4 + med: slurm_med_4 + high: slurm_high_4 + - rule_type: records + nice_value: 0 + lower_bound: 100 + upper_bound: 1000 + destination: + priority: + low: slurm_low_8 + med: slurm_med_8 + high: slurm_high_8 + - rule_type: records + nice_value: 0 + lower_bound: 1000 + upper_bound: 10000 + destination: + priority: + low: slurm_low_8 + med: slurm_med_8 + high: slurm_high_8 + - rule_type: records + nice_value: 0 + lower_bound: 1000000 + upper_bound: Infinity + fail_message: Using the wrong tool + destination: fail + + smalt_map: + rules: + - rule_type: file_size + nice_value: 0 + lower_bound: 0 + upper_bound: 2 GB + destination: + priority: + low: slurm_low_4 + med: slurm_med_4 + high: slurm_high_4 + - rule_type: file_size + nice_value: 0 + lower_bound: 2 GB + upper_bound: 4 GB + destination: + priority: + low: slurm_low_8 + med: slurm_med_8 + high: slurm_high_8 + - rule_type: file_size + nice_value: 0 + lower_bound: 4 GB + upper_bound: Infinity + destination: + priority: + low: slurm_low_16 + med: slurm_med_16 + high: slurm_high_16 + + fastqc: + rules: + - rule_type: file_size + nice_value: 0 + lower_bound: 0 + upper_bound: 400 MB + fail_message: File size too small + destination: fail + - rule_type: file_size + nice_value: 0 + lower_bound: 400 MB + upper_bound: Infinity + destination: + priority: + low: slurm_low_4 + med: slurm_med_4 + high: slurm_high_4 + default_destination: + priority: + low: slurm_low_16 + med: slurm_med_16 + high: slurm_high_16 + +default_destination: + priority: + low: slurm_low_default + med: slurm_med_default + high: slurm_high_default +users: + user@email.com: + priority: high +verbose: True diff --git a/lib/galaxy/config/sample/tool_shed.yml.sample b/lib/galaxy/config/sample/tool_shed.yml.sample new file mode 100644 index 00000000000..59495ea0cd0 --- /dev/null +++ b/lib/galaxy/config/sample/tool_shed.yml.sample @@ -0,0 +1,385 @@ +uwsgi: + + # The address and port on which to listen. By default, only listen to + # localhost (tool_shed will not be accessible over the network). Use + # ':9009' to listen on all available network interfaces. + http: 127.0.0.1:9009 + + # By default uWSGI allocates a very small buffer (4096 bytes) for the + # headers of each request. If you start receiving "invalid request + # block size" in your logs, it could mean you need a bigger buffer. We + # recommend at least 16384. + buffer-size: 16384 + + # Number of web server (worker) processes to fork after the + # application has loaded. If this is set to greater than 1, thunder- + # lock likely should be enabled below. + processes: 1 + + # Number of threads for each web server process. + threads: 4 + + # Number of threads for serving static content and handling internal + # routing requests. + offload-threads: 2 + + # Mapping to serve style content. + static-map: /static/style=static/style/blue + + # Mapping to serve the remainder of the static content. + static-map: /static=static + + # Mapping to serve the favicon. + static-map: /favicon.ico=static/favicon.ico + + # Enable the master process manager. Disabled by default for maximum + # compatibility with CTRL+C, but should be enabled for use with + # --daemon and/or production deployments. + master: false + + # Path to the application's Python virtual environment. If using Conda + # for Galaxy's framework dependencies (not tools!), do not set this. + virtualenv: .venv + + # Path to the application's Python library. + pythonpath: lib + + # The entry point which returns the web application (e.g. Galaxy, + # Reports, etc.) that you are loading. + module: galaxy.webapps.tool_shed.buildapp:uwsgi_app() + + # Mount the web application (e.g. Galaxy, Reports, etc.) at the given + # URL prefix. Cannot be used together with 'module:' above. + #mount: /galaxy=galaxy.webapps.galaxy.buildapp:uwsgi_app() + + # Make uWSGI rewrite PATH_INFO and SCRIPT_NAME according to mount- + # points. Set this to true if a URL prefix is used. + manage-script-name: false + + # It is usually a good idea to set this to ``true`` if processes is + # greater than 1. + thunder-lock: false + + # Cause uWSGI to respect the traditional behavior of dying on SIGTERM + # (its default is to brutally reload workers) + die-on-term: true + + # Cause uWSGI to gracefully reload workers and mules upon receipt of + # SIGINT (its default is to brutally kill workers) + hook-master-start: unix_signal:2 gracefully_kill_them_all + + # Cause uWSGI to gracefully reload workers and mules upon receipt of + # SIGTERM (its default is to brutally kill workers) + hook-master-start: unix_signal:15 gracefully_kill_them_all + + # Feature necessary for proper mule signal handling + py-call-osafterfork: true + + # Ensure application threads will run if `threads` is unset. + enable-threads: true + +tool_shed: + + # Verbosity of console log messages. Acceptable values can be found + # here: https://docs.python.org/library/logging.html#logging-levels + #log_level: DEBUG + + # By default, the Tool Shed uses a SQLite database at + # 'database/community.sqlite'. You may use a SQLAlchemy connection + # string to specify an external database instead. This string takes + # many options which are explained in detail in the config file + # documentation. + #database_connection: sqlite:///./database/community.sqlite?isolation_level=IMMEDIATE + + # Where the hgweb.config file is stored. The default is the Galaxy + # installation directory. + #hgweb_config_dir: null + + # Where Tool Shed repositories are stored. + #file_path: database/community_files + + # Where temporary files are stored. + #new_file_path: database/tmp + + # File containing old-style genome builds + #builds_file_path: tool-data/shared/ucsc/builds.txt + + # Format string used when showing date and time information. The + # string may contain: - the directives used by Python time.strftime() + # function (see + # https://docs.python.org/library/time.html#time.strftime ), - $locale + # (complete format string for the server locale), - $iso8601 (complete + # format string as specified by ISO 8601 international standard). + #pretty_datetime_format: $locale (UTC) + + # -- Repository and Tool search Using the script located at + # scripts/build_ts_whoosh_index.py you can generate search index and + # allow full text API searching over the repositories and tools within + # the Tool Shed given that you specify the following two config + # options. + #toolshed_search_on: true + + # -- Repository and Tool search Using the script located at + # scripts/build_ts_whoosh_index.py you can generate search index and + # allow full text API searching over the repositories and tools within + # the Tool Shed given that you specify the following two config + # options. + #whoosh_index_dir: database/toolshed_whoosh_indexes + + # For searching repositories at /api/repositories: + #repo_name_boost: 0.9 + + # For searching repositories at /api/repositories: + #repo_description_boost: 0.6 + + # For searching repositories at /api/repositories: + #repo_long_description_boost: 0.5 + + # For searching repositories at /api/repositories: + #repo_homepage_url_boost: 0.3 + + # For searching repositories at /api/repositories: + #repo_remote_repository_url_boost: 0.2 + + # For searching repositories at /api/repositories: + #repo_owner_username_boost: 0.3 + + # For searching repositories at /api/repositories: + #categories_boost: 0.5 + + # For searching tools at /api/tools + #tool_name_boost: 1.2 + + # For searching tools at /api/tools + #tool_description_boost: 0.6 + + # For searching tools at /api/tools + #tool_help_boost: 0.4 + + # For searching tools at /api/tools + #tool_repo_owner_username: 0.3 + + # You can enter tracking code here to track visitor's behavior through + # your Google Analytics account. Example: UA-XXXXXXXX-Y + #ga_code: null + + # The Tool Shed encodes various internal values when these values will + # be output in some format (for example, in a URL or cookie). You + # should set a key to be used by the algorithm that encodes and + # decodes these values. It can be any string. If left unchanged, + # anyone could construct a cookie that would grant them access to + # others' sessions. One simple way to generate a value for this is + # with the shell command: python -c 'from __future__ import + # print_function; import time; print(time.time())' | md5sum | cut -f 1 + # -d ' ' + #id_secret: changethisinproductiontoo + + # User authentication can be delegated to an upstream proxy server + # (usually Apache). The upstream proxy should set a REMOTE_USER + # header in the request. Enabling remote user disables regular logins. + # For more information, see: https://galaxyproject.org/admin/config + # /apache-external-user-auth/ + #use_remote_user: false + + # If use_remote_user is enabled, anyone who can log in to the Galaxy + # host may impersonate any other user by simply sending the + # appropriate header. Thus a secret shared between the upstream proxy + # server, and Galaxy is required. If anyone other than the Galaxy user + # is using the server, then apache/nginx should pass a value in the + # header 'GX_SECRET' that is identical the one below + #remote_user_secret: changethisinproductiontoo + + # If use_remote_user is enabled and your external authentication + # method just returns bare usernames, set a default mail domain to be + # appended to usernames, to become your Galaxy usernames (email + # addresses). + #remote_user_maildomain: null + + # If use_remote_user is enabled, the header that the upstream proxy + # provides the remote username in defaults to HTTP_REMOTE_USER (the + # 'HTTP_' is prepended by WSGI). This option allows you to change the + # header. Note, you still need to prepend 'HTTP_' to the header in + # this option, but your proxy server should *not* include 'HTTP_' at + # the beginning of the header name. + #remote_user_header: HTTP_REMOTE_USER + + # If use_remote_user is enabled, you can set this to a URL that will + # log your users out. + #remote_user_logout_href: null + + # Configuration for debugging middleware + #debug: false + + # Check for WSGI compliance. + #use_lint: false + + # Intercept print statements and show them on the returned page. + #use_printdebug: true + + # NEVER enable this on a public site (even test or QA) + #use_interactive: false + + # Administrative users - set this to a comma-separated list of valid + # Tool Shed users (email addresses). These users will have access to + # the Admin section of the server, and will have access to create + # users, groups, roles, libraries, and more. + #admin_users: null + + # Force everyone to log in (disable anonymous access) + #require_login: false + + # Allow unregistered users to create new accounts (otherwise, they + # will have to be created by an admin). + #allow_user_creation: true + + # Allow administrators to delete accounts. + #allow_user_deletion: false + + # For use by email messages sent from the Tool Shed. + #smtp_server: smtp.your_tool_shed_server + + # For use by email messages sent from the Tool Shed. + #email_from: your_tool_shed_email@server + + # If your SMTP server requires a username and password, you can + # provide them here (password in cleartext here, but if your server + # supports STARTTLS it will be sent over the network encrypted). + #smtp_username: null + + # If your SMTP server requires a username and password, you can + # provide them here (password in cleartext here, but if your server + # supports STARTTLS it will be sent over the network encrypted). + #smtp_password: null + + # If your SMTP server requires SSL from the beginning of the + # connection + #smtp_ssl: false + + # The URL linked by the "Support" link in the "Help" menu. + #support_url: https://galaxyproject.org/support/ + + # Address to join mailing list + #mailing_join_addr: galaxy-announce-join@bx.psu.edu + + # Write thread status periodically to 'heartbeat.log' (careful, uses + # disk space rapidly!) + #use_heartbeat: true + + # Profiling middleware (cProfile based) + #use_profile: true + + # Enable creation of Galaxy flavor Docker Image + #enable_galaxy_flavor_docker_image: false + + # Show a message box under the masthead. + #message_box_visible: false + + # Show a message box under the masthead. + #message_box_content: null + + # Class of the message box under the masthead. Possible values are: + # 'info' (the default), 'warning', 'error', 'done'. + #message_box_class: info + + # Serving static files (needed if running standalone) + #static_enabled: true + + # Serving static files (needed if running standalone) + #static_cache_time: 360 + + # Serving static files (needed if running standalone) + #static_dir: static/ + + # Serving static files (needed if running standalone) + #static_images_dir: static/images + + # Serving static files (needed if running standalone) + #static_favicon_dir: static/favicon.ico + + # Serving static files (needed if running standalone) + #static_scripts_dir: static/scripts/ + + # Serving static files (needed if running standalone) + #static_style_dir: static/style/blue + + # Enables GDPR Compliance mode. This makes several changes to the way + # Galaxy logs and exposes data externally such as removing + # emails/usernames from logs and bug reports. You are responsible for + # removing personal data from backups. Please read the GDPR section + # under the special topics area of the admin documentation. + #enable_beta_gdpr: false + + # For help on configuring the Advanced proxy features, see: + # https://docs.galaxyproject.org/en/master/admin/production.html + # Apache can handle file downloads (Galaxy-to-user) via mod_xsendfile. + # Set this to True to inform Galaxy that mod_xsendfile is enabled + # upstream. + #apache_xsendfile: false + + # The same download handling can be done by nginx using X-Accel- + # Redirect. This should be set to the path defined in the nginx + # config as an internal redirect with access to Galaxy's data files + # (see documentation linked above). + #nginx_x_accel_redirect_base: null + + # This value overrides the action set on the file upload form, e.g. + # the web path where the nginx_upload_module has been configured to + # intercept upload requests. + #nginx_upload_path: null + + # E-mail domains blacklist is used for filtering out users that are + # using disposable email address during the registration. If their + # address domain matches any domain in the blacklist, they are refused + # the registration. + #blacklist_file: config/disposable_email_blacklist.conf + + # Append "/{brand}" to the "Galaxy" text in the masthead. + #brand: null + + # Citation related caching. Tool citations information maybe fetched + # from external sources such as https://doi.org/ by Galaxy - the + # following parameters can be used to control the caching used to + # store this information. + #citation_cache_type: file + + # Citation related caching. Tool citations information maybe fetched + # from external sources such as https://doi.org/ by Galaxy - the + # following parameters can be used to control the caching used to + # store this information. + #citation_cache_data_dir: database/citations/data + + # Citation related caching. Tool citations information maybe fetched + # from external sources such as https://doi.org/ by Galaxy - the + # following parameters can be used to control the caching used to + # store this information. + #citation_cache_lock_dir: database/citations/lock + + # Turn on logging of user actions to the database. Actions currently + # logged are grid views, tool searches, and use of "recently" used + # tools menu. The log_events and log_actions functionality will + # eventually be merged. + #log_actions: true + + # Password expiration period (in days). Users are required to change + # their password every x days. Users will be redirected to the change + # password screen when they log in after their password expires. Enter + # 0 to disable password expiration. + #password_expiration_period: 0 + + # Log to Sentry Sentry is an open source logging and error aggregation + # platform. Setting sentry_dsn will enable the Sentry middleware and + # errors will be sent to the indicated sentry instance. This + # connection string is available in your sentry instance under + # -> Settings -> API Keys. + #sentry_dsn: null + + # Galaxy Session Timeout This provides a timeout (in minutes) after + # which a user will have to log back in. A duration of 0 disables this + # feature. + #session_duration: 0 + + # The URL linked by the "Terms and Conditions" link in the "Help" + # menu, as well as on the user registration and login forms and in the + # activation emails. + #terms_url: null + diff --git a/lib/galaxy/config/sample/tool_sheds_conf.xml.sample b/lib/galaxy/config/sample/tool_sheds_conf.xml.sample new file mode 100644 index 00000000000..197cd63e384 --- /dev/null +++ b/lib/galaxy/config/sample/tool_sheds_conf.xml.sample @@ -0,0 +1,7 @@ + + + + + diff --git a/lib/galaxy/config/sample/user_preferences_extra_conf.yml.sample b/lib/galaxy/config/sample/user_preferences_extra_conf.yml.sample new file mode 100644 index 00000000000..2025f845e4b --- /dev/null +++ b/lib/galaxy/config/sample/user_preferences_extra_conf.yml.sample @@ -0,0 +1,65 @@ +preferences: + # the key you can refer to + apollo_url_01: + # description that is displayed to the user + description: The URL to your personal Apollo instance + inputs: + # the content can be accessed by apollo_url_01.apollo_url + - name: apollo_url + label: Apollo URL + # type of input field that will be displayed to the user + # can be string or password + type: text + # by defaul all inputs are required + required: True + - name: apollo_text + label: Apollo Text + # type of input field that will be displayed to the user + # can be string or password + type: text + # by defaul all inputs are required + required: True + + openstack_account: + description: Your own Open Stack account + inputs: + - name: username + label: Username + type: text + required: False + - name: url + label: Open Stack URL + type: text + required: True + + webhook_overlay_search: + description: Configuration option for your search results + inputs: + - name: max_search_results + label: Maximum number of search results + type: text + required: False + + use_cached_job: + description: Do you want to be able to re-use previously run jobs ? + inputs: + - name: use_cached_job_checkbox + label: Do you want to be able to re-use equivalent jobs ? + type: boolean + checked: false + value: false + help: If you select yes, you will be able to select for each tool and workflow run if you would like to use this feature. + + localization: + description: Localization + inputs: + - name: locale + label: Prefered language + type: select + required: False + options: + - [Navigator default, auto] + - [Chinese, zh] + - [English, en] + - [French, fr] + - [Japanese, ja] diff --git a/lib/galaxy/config/sample/workflow_resource_mapper_conf.yml.sample b/lib/galaxy/config/sample/workflow_resource_mapper_conf.yml.sample new file mode 100644 index 00000000000..648dd09ddee --- /dev/null +++ b/lib/galaxy/config/sample/workflow_resource_mapper_conf.yml.sample @@ -0,0 +1,14 @@ +by_group: + default: default + groups: + default: [project, priority] + prio_basic: [{name: priority, options: ["low", "med"]}] + prio_advanced: [{name: priority, options: ["low", "med", "high"]}] + prio_super: + - time + - memory + - processors + - name: priority + options: + - ultra + - plus_ultra diff --git a/lib/galaxy/config/sample/workflow_resource_params_conf.xml.sample b/lib/galaxy/config/sample/workflow_resource_params_conf.xml.sample new file mode 100644 index 00000000000..0495c2443ed --- /dev/null +++ b/lib/galaxy/config/sample/workflow_resource_params_conf.xml.sample @@ -0,0 +1,13 @@ + + + + + + + diff --git a/lib/galaxy/config/sample/workflow_schedulers_conf.xml.sample b/lib/galaxy/config/sample/workflow_schedulers_conf.xml.sample new file mode 100644 index 00000000000..1c83136f870 --- /dev/null +++ b/lib/galaxy/config/sample/workflow_schedulers_conf.xml.sample @@ -0,0 +1,42 @@ + + + + + + + + + + + + diff --git a/lib/galaxy/config/script.py b/lib/galaxy/config/script.py new file mode 100644 index 00000000000..6cf211a890d --- /dev/null +++ b/lib/galaxy/config/script.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python +from __future__ import print_function + +import os +import string +import sys +from argparse import ArgumentParser + +try: + import pip +except ImportError: + pip = None + + +CONFIGURE_URL = "https://wiki.galaxyproject.org/Admin/Config/Performance/ProductionServer" + +DESCRIPTION = "Initialize a directory with a minimal Galaxy config." +HELP_CONFIG_DIR = "Directory containing the configuration files for Galaxy." +HELP_DATA_DIR = "Directory containing Galaxy-created data." +HELP_FORCE = "Overwrite existing files if they already exist." +HELP_WSGI_SERVER = ("Web server stack used to host Galaxy web application, and " + "if uWSGI, which protocol to use.") +HELP_LIBDRMAA = ("Configure Galaxy to submit jobs to a cluster via DRMAA by " + "supplying the path to a libdrmaa.so file using this argument.") +HELP_INSTALL = ("Install optional dependencies required by specified configuration " + "(e.g. drmaa, uwsgi, etc...).") +HELP_HOST = ('Host to bind Galaxy to - defaults to localhost. Specify an IP ' + 'address or "all" to listen on all interfaces.') +HELP_PORT = ("Port to bind Galaxy to.") +HELP_DB_CONN = ('Galaxy database connection URI.') + +DEFAULT_HOST = "localhost" +DEFAULT_INI = "galaxy.ini" +DEFAULT_DB_CONN = 'sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE' + +SAMPLES_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), 'sample')) +GALAXY_CONFIG_TEMPLATE_FILE = os.path.join(SAMPLES_PATH, 'galaxy.ini.sample') +STATIC_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'web', 'framework', 'static')) + +# The sample is used as the default config file for Galaxy started without a +# config, and we don't want to duplicate the whole thing into galaxy.config for +# templating, so for now just substitute some lines. In the future we will +# build configs differently. +GALAXY_CONFIG_SUBSTITUTIONS = { + '#port = 8080': 'port = ${port}', + '#host = 127.0.0.1': 'host = ${host}', + 'http = 127.0.0.1:8080': '${uwsgi_transport} = ${host}:${port}', + 'static-map = /static=/path/to/galaxy/web/framework/static': 'static-map = /static=${static_path}', + 'static-map = /static/style=/path/to/galaxy/web/framework/static/style/blue': 'static-map = /static/style=${static_path}/style/blue', + '#config_dir = None': 'config_dir = ${config_dir}', + '#data_dir = None': 'data_dir = ${data_dir}', + '#database_connection = sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE': 'database_connection = ${database_connection}', +} + + +def main(argv=None): + dependencies = [] + arg_parser = ArgumentParser(description=DESCRIPTION) + arg_parser.add_argument("--config-dir", + default=".", + help=HELP_CONFIG_DIR) + arg_parser.add_argument("--data-dir", + default="./data", + help=HELP_DATA_DIR) + arg_parser.add_argument("--wsgi-server", + choices=["paster", "uwsgi-http", "uwsgi-native"], + default="paster", + help=HELP_WSGI_SERVER) + arg_parser.add_argument("--host", + default=DEFAULT_HOST, + help=HELP_HOST) + arg_parser.add_argument("--port", + default="8080", + help=HELP_PORT) + arg_parser.add_argument("--db-conn", + default=DEFAULT_DB_CONN, + help=HELP_DB_CONN) + arg_parser.add_argument("--install", + action="store_true", + help=HELP_INSTALL) + arg_parser.add_argument("--force", + action="store_true", + default=False, + help=HELP_FORCE) + args = arg_parser.parse_args(argv) + config_dir = args.config_dir + relative_config_dir = config_dir + config_dir = os.path.abspath(config_dir) + data_dir = args.data_dir + data_dir = os.path.abspath(data_dir) + + mode = _determine_mode(args) + if mode.startswith("uwsgi-"): + dependencies.append("uwsgi") + + for directory in (config_dir, data_dir): + if not os.path.exists(directory): + os.makedirs(directory) + + print("Bootstrapping Galaxy configuration into directory %s" % relative_config_dir) + _handle_galaxy_ini(args, config_dir, data_dir) + _handle_install(args, dependencies) + _print_config_summary(args, mode, relative_config_dir) + + +def _print_config_summary(args, mode, relative_config_dir): + _print_galaxy_ini_info(args, mode) + print("") + print("For help on configuring Galaxy, consult the documentation at:\n ", CONFIGURE_URL) + print("") + print("Additional sample configuration files for various Galaxy components (jobs,") + print("datatypes, etc.) can be found in:\n ", SAMPLES_PATH) + print("") + print("Start Galaxy by running the command from directory [%s]:" % relative_config_dir) + _print_galaxy_run(mode) + + +def _print_galaxy_ini_info(args, mode): + print(" - galaxy.ini created, update to configure Galaxy.") + print(" * Target web server %s" % mode) + if args.host == DEFAULT_HOST: + print(" * Binding to host localhost, remote clients will not be able to connect.") + elif _determine_host(args) == "0.0.0.0": + print(" * Binding to all network interfaces.") + else: + print(" * Binding to host [%s].", args.host) + + +def _print_galaxy_run(mode): + if mode.startswith("uwsgi"): + print(" uwsgi --ini-paste galaxy.ini") + else: + print(" galaxy-paster serve galaxy.ini") + + +def _determine_mode(args): + if args.wsgi_server: + mode = args.wsgi_server + else: + mode = "paster" + return mode + + +def _determine_host(args): + return '0.0.0.0' if args.host == 'all' else args.host + + +def _determine_ini_file(config_dir): + return os.path.join(config_dir, DEFAULT_INI) + + +def _handle_galaxy_ini(args, config_dir, data_dir): + force = args.force + ini_file = _determine_ini_file(config_dir) + _check_file(ini_file, force) + uwsgi_transport = 'socket' if args.wsgi_server == 'uwsgi-native' else 'http' + config_dict = dict( + port=args.port, + host=_determine_host(args), + uwsgi_transport=uwsgi_transport, + config_dir=config_dir, + data_dir=data_dir, + static_path=STATIC_PATH, + database_connection=args.db_conn, + ) + + galaxy_config_template = [] + with open(GALAXY_CONFIG_TEMPLATE_FILE) as fh: + for line in [l.rstrip('\n') for l in fh.readlines()]: + for k, v in GALAXY_CONFIG_SUBSTITUTIONS.items(): + if line == k: + line = v + galaxy_config_template.append(line) + galaxy_config_template = string.Template('\n'.join(galaxy_config_template)) + + galaxy_config = galaxy_config_template.safe_substitute( + **config_dict + ) + open(ini_file, "w").write(galaxy_config) + + +def _handle_install(args, dependencies): + if args.install and dependencies: + if pip is None: + raise ImportError("Bootstrapping Pulsar dependencies requires pip library.") + + pip.main(["install"] + dependencies) + + +def _check_file(path, force): + if os.path.exists(path) and not force: + print("File %s exists, exiting. Run with --force to replace configuration." % path, file=sys.stderr) + sys.exit(1) diff --git a/lib/galaxy/webapps/galaxy/config_watchers.py b/lib/galaxy/config_watchers.py similarity index 100% rename from lib/galaxy/webapps/galaxy/config_watchers.py rename to lib/galaxy/config_watchers.py diff --git a/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py b/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py index 98e6af16c01..1b893f1ab5d 100644 --- a/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py +++ b/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py @@ -6,7 +6,7 @@ import sys import bx.align.maf -from galaxy.tools.util import maf_utilities +from galaxy.datatypes.util import maf_utilities assert sys.version_info[:2] >= (2, 6) diff --git a/lib/galaxy/datatypes/converters/maf_to_interval_converter.py b/lib/galaxy/datatypes/converters/maf_to_interval_converter.py index 51049ed54c6..a19ab25cabb 100644 --- a/lib/galaxy/datatypes/converters/maf_to_interval_converter.py +++ b/lib/galaxy/datatypes/converters/maf_to_interval_converter.py @@ -6,7 +6,7 @@ import sys import bx.align.maf -from galaxy.tools.util import maf_utilities +from galaxy.datatypes.util import maf_utilities assert sys.version_info[:2] >= (2, 6) diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 7a4261f5378..bdf6ff9fac5 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -123,12 +123,24 @@ class Registry(object): if not self.converters_path: self.converters_path_attr = registration.get('converters_path', 'lib/galaxy/datatypes/converters') self.converters_path = os.path.join(root_dir, self.converters_path_attr) + if self.converters_path_attr == 'lib/galaxy/datatypes/converters' \ + and not os.path.isdir(self.converters_path): + # Deal with the old default of this path being set in + # datatypes_conf.xml.sample (this path is not useful in an + # "installed Galaxy" world) + self.converters_path_attr = os.path.abspath(os.path.join(os.path.dirname(__file__), 'converters')) + self.converters_path = self.converters_path_attr if not os.path.isdir(self.converters_path): raise ConfigurationError("Directory does not exist: %s" % self.converters_path) if use_display_applications: if not self.display_applications_path: self.display_path_attr = registration.get('display_path', 'display_applications') self.display_applications_path = os.path.join(root_dir, self.display_path_attr) + if self.display_path_attr == 'display_applications' \ + and not os.path.isdir('display_applications'): + # Ditto as with converters_path + self.display_path_attr = os.path.abspath(os.path.join(os.path.dirname(__file__), 'display_applications', 'configs')) + self.display_applications_path = self.display_path_attr # Proprietary datatype's tag may have special attributes, proprietary_converter_path and proprietary_display_path. proprietary_converter_path = registration.get('proprietary_converter_path', None) proprietary_display_path = registration.get('proprietary_display_path', None) @@ -734,7 +746,7 @@ class Registry(object): # We need to be able to add a job to the queue to set metadata. The queue will currently only accept jobs with an associated # tool. We'll load a special tool to be used for Auto-Detecting metadata; this is less than ideal, but effective # Properly building a tool without relying on parsing an XML file is near difficult...so we bundle with Galaxy. - set_meta_tool = toolbox.load_hidden_lib_tool("galaxy/datatypes/set_metadata_tool.xml") + set_meta_tool = toolbox.load_hidden_lib_tool(os.path.abspath(os.path.join(os.path.dirname(__file__), "set_metadata_tool.xml"))) self.set_external_metadata_tool = set_meta_tool self.log.debug("Loaded external metadata tool: %s", self.set_external_metadata_tool.id) @@ -981,7 +993,7 @@ class Registry(object): def example_datatype_registry_for_sample(sniff_compressed_dynamic_datatypes_default=True): galaxy_dir = galaxy.util.galaxy_directory() - sample_conf = os.path.join(galaxy_dir, "config", "datatypes_conf.xml.sample") + sample_conf = os.path.join(galaxy_dir, "lib", "galaxy", "config", "sample", "datatypes_conf.xml.sample") config = Bunch(sniff_compressed_dynamic_datatypes_default=sniff_compressed_dynamic_datatypes_default) datatypes_registry = Registry(config) datatypes_registry.load_datatypes(root_dir=galaxy_dir, config=sample_conf) diff --git a/lib/galaxy/datatypes/util/maf_utilities.py b/lib/galaxy/datatypes/util/maf_utilities.py new file mode 100644 index 00000000000..3a5aa531bb2 --- /dev/null +++ b/lib/galaxy/datatypes/util/maf_utilities.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python +""" +Provides wrappers and utilities for working with MAF files and alignments. +""" +# Dan Blankenberg +from __future__ import print_function + +import functools +import logging +import os +import resource +import sys +import tempfile +from copy import deepcopy +from errno import EMFILE + +import bx.align.maf +import bx.interval_index_file +import bx.intervals +from six.moves import xrange + +try: + from string import maketrans +except ImportError: + maketrans = str.maketrans + +assert sys.version_info[:2] >= (2, 6) + +log = logging.getLogger(__name__) + +GAP_CHARS = ['-'] +SRC_SPLIT_CHAR = '.' + + +def src_split(src): + fields = src.split(SRC_SPLIT_CHAR, 1) + spec = fields.pop(0) + if fields: + chrom = fields.pop(0) + else: + chrom = spec + return spec, chrom + + +def src_merge(spec, chrom, contig=None): + if None in [spec, chrom]: + spec = chrom = spec or chrom + return bx.align.maf.src_merge(spec, chrom, contig) + + +def get_species_in_block(block): + species = [] + for c in block.components: + spec, chrom = src_split(c.src) + if spec not in species: + species.append(spec) + return species + + +def tool_fail(msg="Unknown Error"): + print("Fatal Error: %s" % msg, file=sys.stderr) + sys.exit() + + +class TempFileHandler(object): + ''' + Handles creating, opening, closing, and deleting of Temp files, with a + maximum number of files open at one time. + ''' + + DEFAULT_MAX_OPEN_FILES = max(resource.getrlimit(resource.RLIMIT_NOFILE)[0] / 2, 1) + + def __init__(self, max_open_files=None, **kwds): + if max_open_files is None: + max_open_files = self.DEFAULT_MAX_OPEN_FILES + self.max_open_files = max_open_files + self.files = [] + self.open_file_indexes = [] + self.kwds = kwds + + def get_open_tempfile(self, index=None, **kwds): + if index is not None and index in self.open_file_indexes: + self.open_file_indexes.remove(index) + else: + if self.max_open_files: + while len(self.open_file_indexes) >= self.max_open_files: + self.close(self.open_file_indexes[0]) + if index is None: + index = len(self.files) + temp_kwds = dict(self.kwds) + temp_kwds.update(kwds) + # Being able to use delete=True here, would simplify a bit, + # but we support python2.4 in these tools + while True: + try: + tmp_file = tempfile.NamedTemporaryFile(**temp_kwds) + filename = tmp_file.name + break + except OSError as e: + if self.open_file_indexes and e.errno == EMFILE: + self.max_open_files = len(self.open_file_indexes) + self.close(self.open_file_indexes[0]) + else: + raise e + tmp_file.close() + self.files.append(open(filename, 'w+b')) + else: + while True: + try: + self.files[index] = open(self.files[index].name, 'r+b') + break + except OSError as e: + if self.open_file_indexes and e.errno == EMFILE: + self.max_open_files = len(self.open_file_indexes) + self.close(self.open_file_indexes[0]) + else: + raise e + self.files[index].seek(0, 2) + self.open_file_indexes.append(index) + return index, self.files[index] + + def close(self, index, delete=False): + if index in self.open_file_indexes: + self.open_file_indexes.remove(index) + rval = self.files[index].close() + if delete: + try: + os.unlink(self.files[index].name) + except OSError: + pass + return rval + + def flush(self, index): + if index in self.open_file_indexes: + self.files[index].flush() + + def __del__(self): + for i in xrange(len(self.files)): + self.close(i, delete=True) + + +# an object corresponding to a reference layered alignment +class RegionAlignment(object): + + DNA_COMPLEMENT = maketrans("ACGTacgt", "TGCAtgca") + MAX_SEQUENCE_SIZE = sys.maxsize # Maximum length of sequence allowed + + def __init__(self, size, species=[], temp_file_handler=None): + assert size <= self.MAX_SEQUENCE_SIZE, "Maximum length allowed for an individual sequence has been exceeded (%i > %i)." % (size, self.MAX_SEQUENCE_SIZE) + self.size = size + if not temp_file_handler: + temp_file_handler = TempFileHandler() + self.temp_file_handler = temp_file_handler + self.sequences = {} + if not isinstance(species, list): + species = [species] + for spec in species: + self.add_species(spec) + + # add a species to the alignment + def add_species(self, species): + # make temporary sequence files + file_index, fh = self.temp_file_handler.get_open_tempfile() + self.sequences[species] = file_index + fh.write("-" * self.size) + + # returns the names for species found in alignment, skipping names as requested + def get_species_names(self, skip=[]): + if not isinstance(skip, list): + skip = [skip] + names = list(self.sequences.keys()) + for name in skip: + try: + names.remove(name) + except ValueError: + pass + return names + + # returns the sequence for a species + def get_sequence(self, species): + file_index, fh = self.temp_file_handler.get_open_tempfile(self.sequences[species]) + fh.seek(0) + return fh.read() + + # returns the reverse complement of the sequence for a species + def get_sequence_reverse_complement(self, species): + complement = [base for base in self.get_sequence(species).translate(self.DNA_COMPLEMENT)] + complement.reverse() + return "".join(complement) + + # sets a position for a species + def set_position(self, index, species, base): + if len(base) != 1: + raise Exception("A genomic position can only have a length of 1.") + return self.set_range(index, species, base) + # sets a range for a species + + def set_range(self, index, species, bases): + if index >= self.size or index < 0: + raise Exception("Your index (%i) is out of range (0 - %i)." % (index, self.size - 1)) + if len(bases) == 0: + raise Exception("A set of genomic positions can only have a positive length.") + if species not in self.sequences.keys(): + self.add_species(species) + file_index, fh = self.temp_file_handler.get_open_tempfile(self.sequences[species]) + fh.seek(index) + fh.write(bases) + + # Flush temp file of specified species, or all species + def flush(self, species=None): + if species is None: + species = self.sequences.keys() + elif not isinstance(species, list): + species = [species] + for spec in species: + self.temp_file_handler.flush(self.sequences[spec]) + + +class GenomicRegionAlignment(RegionAlignment): + + def __init__(self, start, end, species=[], temp_file_handler=None): + RegionAlignment.__init__(self, end - start, species, temp_file_handler=temp_file_handler) + self.start = start + self.end = end + + +class SplicedAlignment(object): + + DNA_COMPLEMENT = maketrans("ACGTacgt", "TGCAtgca") + + def __init__(self, exon_starts, exon_ends, species=[], temp_file_handler=None): + if not isinstance(exon_starts, list): + exon_starts = [exon_starts] + if not isinstance(exon_ends, list): + exon_ends = [exon_ends] + assert len(exon_starts) == len(exon_ends), "The number of starts does not match the number of sizes." + self.exons = [] + if not temp_file_handler: + temp_file_handler = TempFileHandler() + self.temp_file_handler = temp_file_handler + for i in range(len(exon_starts)): + self.exons.append(GenomicRegionAlignment(exon_starts[i], exon_ends[i], species, temp_file_handler=temp_file_handler)) + + # returns the names for species found in alignment, skipping names as requested + def get_species_names(self, skip=[]): + if not isinstance(skip, list): + skip = [skip] + names = [] + for exon in self.exons: + for name in exon.get_species_names(skip=skip): + if name not in names: + names.append(name) + return names + + # returns the sequence for a species + def get_sequence(self, species): + index, fh = self.temp_file_handler.get_open_tempfile() + for exon in self.exons: + if species in exon.get_species_names(): + seq = exon.get_sequence(species) + # we need to refetch fh here, since exon.get_sequence( species ) uses a tempfile + # and if max==1, it would close fh + index, fh = self.temp_file_handler.get_open_tempfile(index) + fh.write(seq) + else: + fh.write("-" * exon.size) + fh.seek(0) + rval = fh.read() + self.temp_file_handler.close(index, delete=True) + return rval + + # returns the reverse complement of the sequence for a species + def get_sequence_reverse_complement(self, species): + complement = [base for base in self.get_sequence(species).translate(self.DNA_COMPLEMENT)] + complement.reverse() + return "".join(complement) + + # Start and end of coding region + @property + def start(self): + return self.exons[0].start + + @property + def end(self): + return self.exons[-1].end + + +# Open a MAF index using a UID +def maf_index_by_uid(maf_uid, index_location_file): + for line in open(index_location_file): + try: + # read each line, if not enough fields, go to next line + if line[0:1] == "#": + continue + fields = line.split('\t') + if maf_uid == fields[1]: + try: + maf_files = fields[4].replace("\n", "").replace("\r", "").split(",") + return bx.align.maf.MultiIndexed(maf_files, keep_open=True, parse_e_rows=False) + except Exception as e: + raise Exception('MAF UID (%s) found, but configuration appears to be malformed: %s' % (maf_uid, e)) + except Exception: + pass + return None + + +# return ( index, temp_index_filename ) for user maf, if available, or build one and return it, return None when no tempfile is created +def open_or_build_maf_index(maf_file, index_filename, species=None): + try: + return (bx.align.maf.Indexed(maf_file, index_filename=index_filename, keep_open=True, parse_e_rows=False), None) + except Exception: + return build_maf_index(maf_file, species=species) + + +def build_maf_index_species_chromosomes(filename, index_species=None): + species = [] + species_chromosomes = {} + indexes = bx.interval_index_file.Indexes() + blocks = 0 + try: + maf_reader = bx.align.maf.Reader(open(filename)) + while True: + pos = maf_reader.file.tell() + block = next(maf_reader) + if block is None: + break + blocks += 1 + for c in block.components: + spec = c.src + chrom = None + if "." in spec: + spec, chrom = spec.split(".", 1) + if spec not in species: + species.append(spec) + species_chromosomes[spec] = [] + if chrom and chrom not in species_chromosomes[spec]: + species_chromosomes[spec].append(chrom) + if index_species is None or spec in index_species: + forward_strand_start = c.forward_strand_start + forward_strand_end = c.forward_strand_end + try: + forward_strand_start = int(forward_strand_start) + forward_strand_end = int(forward_strand_end) + except ValueError: + continue # start and end are not integers, can't add component to index, goto next component + # this likely only occurs when parse_e_rows is True? + # could a species exist as only e rows? should the + if forward_strand_end > forward_strand_start: + # require positive length; i.e. certain lines have start = end = 0 and cannot be indexed + indexes.add(c.src, forward_strand_start, forward_strand_end, pos, max=c.src_size) + except Exception as e: + # most likely a bad MAF + log.debug('Building MAF index on %s failed: %s' % (filename, e)) + return (None, [], {}, 0) + return (indexes, species, species_chromosomes, blocks) + + +# builds and returns ( index, index_filename ) for specified maf_file +def build_maf_index(maf_file, species=None): + indexes, found_species, species_chromosomes, blocks = build_maf_index_species_chromosomes(maf_file, species) + if indexes is not None: + fd, index_filename = tempfile.mkstemp() + out = os.fdopen(fd, 'w') + indexes.write(out) + out.close() + return (bx.align.maf.Indexed(maf_file, index_filename=index_filename, keep_open=True, parse_e_rows=False), index_filename) + return (None, None) + + +def component_overlaps_region(c, region): + if c is None: + return False + start, end = c.get_forward_strand_start(), c.get_forward_strand_end() + if region.start >= end or region.end <= start: + return False + return True + + +def chop_block_by_region(block, src, region, species=None, mincols=0): + # This chopping method was designed to maintain consistency with how start/end padding gaps have been working in Galaxy thus far: + # behavior as seen when forcing blocks to be '+' relative to src sequence (ref) and using block.slice_by_component( ref, slice_start, slice_end ) + # whether-or-not this is the 'correct' behavior is questionable, but this will at least maintain consistency + # comments welcome + slice_start = block.text_size # max for the min() + slice_end = 0 # min for the max() + old_score = block.score # save old score for later use + # We no longer assume only one occurance of src per block, so we need to check them all + for c in iter_components_by_src(block, src): + if component_overlaps_region(c, region): + if c.text is not None: + rev_strand = False + if c.strand == "-": + # We want our coord_to_col coordinates to be returned from positive stranded component + rev_strand = True + c = c.reverse_complement() + start = max(region.start, c.start) + end = min(region.end, c.end) + start = c.coord_to_col(start) + end = c.coord_to_col(end) + if rev_strand: + # need to orient slice coordinates to the original block direction + slice_len = end - start + end = len(c.text) - start + start = end - slice_len + slice_start = min(start, slice_start) + slice_end = max(end, slice_end) + + if slice_start < slice_end: + block = block.slice(slice_start, slice_end) + if block.text_size > mincols: + # restore old score, may not be accurate, but it is better than 0 for everything? + block.score = old_score + if species is not None: + block = block.limit_to_species(species) + block.remove_all_gap_columns() + return block + return None + + +def orient_block_by_region(block, src, region, force_strand=None): + # loop through components matching src, + # make sure each of these components overlap region + # cache strand for each of overlaping regions + # if force_strand / region.strand not in strand cache, reverse complement + # we could have 2 sequences with same src, overlapping region, on different strands, this would cause no reverse_complementing + strands = [c.strand for c in iter_components_by_src(block, src) if component_overlaps_region(c, region)] + if strands and (force_strand is None and region.strand not in strands) or (force_strand is not None and force_strand not in strands): + block = block.reverse_complement() + return block + + +def get_oriented_chopped_blocks_for_region(index, src, region, species=None, mincols=0, force_strand=None): + for block, idx, offset in get_oriented_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols, force_strand): + yield block + + +def get_oriented_chopped_blocks_with_index_offset_for_region(index, src, region, species=None, mincols=0, force_strand=None): + for block, idx, offset in get_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols): + yield orient_block_by_region(block, src, region, force_strand), idx, offset + + +# split a block with multiple occurances of src into one block per src +def iter_blocks_split_by_src(block, src): + for src_c in iter_components_by_src(block, src): + new_block = bx.align.Alignment(score=block.score, attributes=deepcopy(block.attributes)) + new_block.text_size = block.text_size + for c in block.components: + if c == src_c or c.src != src: + new_block.add_component(deepcopy(c)) # components have reference to alignment, don't want to lose reference to original alignment block in original components + yield new_block + + +# split a block into multiple blocks with all combinations of a species appearing only once per block +def iter_blocks_split_by_species(block, species=None): + def __split_components_by_species(components_by_species, new_block): + if components_by_species: + # more species with components to add to this block + components_by_species = deepcopy(components_by_species) + spec_comps = components_by_species.pop(0) + for c in spec_comps: + newer_block = deepcopy(new_block) + newer_block.add_component(deepcopy(c)) + for value in __split_components_by_species(components_by_species, newer_block): + yield value + else: + # no more components to add, yield this block + yield new_block + + # divide components by species + spec_dict = {} + if not species: + species = [] + for c in block.components: + spec, chrom = src_split(c.src) + if spec not in spec_dict: + spec_dict[spec] = [] + species.append(spec) + spec_dict[spec].append(c) + else: + for spec in species: + spec_dict[spec] = [] + for c in iter_components_by_src_start(block, spec): + spec_dict[spec].append(c) + + empty_block = bx.align.Alignment(score=block.score, attributes=deepcopy(block.attributes)) # should we copy attributes? + empty_block.text_size = block.text_size + # call recursive function to split into each combo of spec/blocks + for value in __split_components_by_species(list(spec_dict.values()), empty_block): + sort_block_components_by_block(value, block) # restore original component order + yield value + + +# generator yielding only chopped and valid blocks for a specified region +def get_chopped_blocks_for_region(index, src, region, species=None, mincols=0): + for block, idx, offset in get_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols): + yield block + + +def get_chopped_blocks_with_index_offset_for_region(index, src, region, species=None, mincols=0): + for block, idx, offset in index.get_as_iterator_with_index_and_offset(src, region.start, region.end): + block = chop_block_by_region(block, src, region, species, mincols) + if block is not None: + yield block, idx, offset + + +# returns a filled region alignment for specified regions +def get_region_alignment(index, primary_species, chrom, start, end, strand='+', species=None, mincols=0, overwrite_with_gaps=True, temp_file_handler=None): + if species is not None: + alignment = RegionAlignment(end - start, species, temp_file_handler=temp_file_handler) + else: + alignment = RegionAlignment(end - start, primary_species, temp_file_handler=temp_file_handler) + return fill_region_alignment(alignment, index, primary_species, chrom, start, end, strand, species, mincols, overwrite_with_gaps) + + +# reduces a block to only positions exisiting in the src provided +def reduce_block_by_primary_genome(block, species, chromosome, region_start): + # returns ( startIndex, {species:texts} + # where texts' contents are reduced to only positions existing in the primary genome + src = "%s.%s" % (species, chromosome) + ref = block.get_component_by_src(src) + start_offset = ref.start - region_start + species_texts = {} + for c in block.components: + species_texts[c.src.split('.')[0]] = list(c.text) + # remove locations which are gaps in the primary species, starting from the downstream end + for i in range(len(species_texts[species]) - 1, -1, -1): + if species_texts[species][i] == '-': + for text in species_texts.values(): + text.pop(i) + for spec, text in species_texts.items(): + species_texts[spec] = ''.join(text) + return (start_offset, species_texts) + + +# fills a region alignment +def fill_region_alignment(alignment, index, primary_species, chrom, start, end, strand='+', species=None, mincols=0, overwrite_with_gaps=True): + region = bx.intervals.Interval(start, end) + region.chrom = chrom + region.strand = strand + primary_src = "%s.%s" % (primary_species, chrom) + + # Order blocks overlaping this position by score, lowest first + blocks = [] + for block, idx, offset in index.get_as_iterator_with_index_and_offset(primary_src, start, end): + score = float(block.score) + for i in range(0, len(blocks)): + if score < blocks[i][0]: + blocks.insert(i, (score, idx, offset)) + break + else: + blocks.append((score, idx, offset)) + + # gap_chars_tuple = tuple( GAP_CHARS ) + gap_chars_str = ''.join(GAP_CHARS) + # Loop through ordered blocks and layer by increasing score + for block_dict in blocks: + for block in iter_blocks_split_by_species(block_dict[1].get_at_offset(block_dict[2])): # need to handle each occurance of sequence in block seperately + if component_overlaps_region(block.get_component_by_src(primary_src), region): + block = chop_block_by_region(block, primary_src, region, species, mincols) # chop block + block = orient_block_by_region(block, primary_src, region) # orient block + start_offset, species_texts = reduce_block_by_primary_genome(block, primary_species, chrom, start) + for spec, text in species_texts.items(): + # we should trim gaps from both sides, since these are not positions in this species genome (sequence) + text = text.rstrip(gap_chars_str) + gap_offset = 0 + # while text.startswith( gap_chars_tuple ): + while True in [text.startswith(gap_char) for gap_char in GAP_CHARS]: # python2.4 doesn't accept a tuple for .startswith() + gap_offset += 1 + text = text[1:] + if not text: + break + if text: + if overwrite_with_gaps: + alignment.set_range(start_offset + gap_offset, spec, text) + else: + for i, char in enumerate(text): + if char not in GAP_CHARS: + alignment.set_position(start_offset + gap_offset + i, spec, char) + return alignment + + +# returns a filled spliced region alignment for specified region with start and end lists +def get_spliced_region_alignment(index, primary_species, chrom, starts, ends, strand='+', species=None, mincols=0, overwrite_with_gaps=True, temp_file_handler=None): + # create spliced alignment object + if species is not None: + alignment = SplicedAlignment(starts, ends, species, temp_file_handler=temp_file_handler) + else: + alignment = SplicedAlignment(starts, ends, [primary_species], temp_file_handler=temp_file_handler) + for exon in alignment.exons: + fill_region_alignment(exon, index, primary_species, chrom, exon.start, exon.end, strand, species, mincols, overwrite_with_gaps) + return alignment + + +# loop through string array, only return non-commented lines +def line_enumerator(lines, comment_start='#'): + i = 0 + for line in lines: + if not line.startswith(comment_start): + i += 1 + yield (i, line) + + +# read a GeneBed file, return list of starts, ends, raw fields +def get_starts_ends_fields_from_gene_bed(line): + # Starts and ends for exons + starts = [] + ends = [] + + fields = line.split() + # Requires atleast 12 BED columns + if len(fields) < 12: + raise Exception("Not a proper 12 column BED line (%s)." % line) + tx_start = int(fields[1]) + strand = fields[5] + if strand != '-': + strand = '+' # Default strand is + + cds_start = int(fields[6]) + cds_end = int(fields[7]) + + # Calculate and store starts and ends of coding exons + region_start, region_end = cds_start, cds_end + exon_starts = list(map(int, fields[11].rstrip(',\n').split(','))) + exon_starts = [x + tx_start for x in exon_starts] + exon_ends = list(map(int, fields[10].rstrip(',').split(','))) + exon_ends = [x + y for x, y in zip(exon_starts, exon_ends)] + for start, end in zip(exon_starts, exon_ends): + start = max(start, region_start) + end = min(end, region_end) + if start < end: + starts.append(start) + ends.append(end) + return (starts, ends, fields) + + +def iter_components_by_src(block, src): + for c in block.components: + if c.src == src: + yield c + + +def get_components_by_src(block, src): + return [value for value in iter_components_by_src(block, src)] + + +def iter_components_by_src_start(block, src): + for c in block.components: + if c.src.startswith(src): + yield c + + +def get_components_by_src_start(block, src): + return [value for value in iter_components_by_src_start(block, src)] + + +def sort_block_components_by_block(block1, block2): + # orders the components in block1 by the index of the component in block2 + # block1 must be a subset of block2 + # occurs in-place + return block1.components.sort(key=functools.cmp_to_key(lambda x, y: block2.components.index(x) - block2.components.index(y))) + + +def get_species_in_maf(maf_filename): + species = [] + for block in bx.align.maf.Reader(open(maf_filename)): + for spec in get_species_in_block(block): + if spec not in species: + species.append(spec) + return species + + +def parse_species_option(species): + if species: + species = species.split(',') + if 'None' not in species: + return species + return None # provided species was '', None, or had 'None' in it + + +def remove_temp_index_file(index_filename): + try: + os.unlink(index_filename) + except Exception: + pass + +# Below are methods to deal with FASTA files + + +def get_fasta_header(component, attributes={}, suffix=None): + header = ">%s(%s):%i-%i|" % (component.src, component.strand, component.get_forward_strand_start(), component.get_forward_strand_end()) + for key, value in attributes.items(): + header = "%s%s=%s|" % (header, key, value) + if suffix: + header = "%s%s" % (header, suffix) + else: + header = "%s%s" % (header, src_split(component.src)[0]) + return header + + +def get_attributes_from_fasta_header(header): + if not header: + return {} + attributes = {} + header = header.lstrip('>') + header = header.strip() + fields = header.split('|') + try: + region = fields[0] + region = region.split('(', 1) + temp = region[0].split('.', 1) + attributes['species'] = temp[0] + if len(temp) == 2: + attributes['chrom'] = temp[1] + else: + attributes['chrom'] = temp[0] + region = region[1].split(')', 1) + attributes['strand'] = region[0] + region = region[1].lstrip(':').split('-') + attributes['start'] = int(region[0]) + attributes['end'] = int(region[1]) + except Exception: + # fields 0 is not a region coordinate + pass + if len(fields) > 2: + for i in range(1, len(fields) - 1): + prop = fields[i].split('=', 1) + if len(prop) == 2: + attributes[prop[0]] = prop[1] + if len(fields) > 1: + attributes['__suffix__'] = fields[-1] + return attributes + + +def iter_fasta_alignment(filename): + class fastaComponent(object): + def __init__(self, species, text=""): + self.species = species + self.text = text + + def extend(self, text): + self.text = self.text + text.replace('\n', '').replace('\r', '').strip() + # yields a list of fastaComponents for a FASTA file + f = open(filename, 'rb') + components = [] + # cur_component = None + while True: + line = f.readline() + if not line: + if components: + yield components + return + line = line.strip() + if not line: + if components: + yield components + components = [] + elif line.startswith('>'): + attributes = get_attributes_from_fasta_header(line) + components.append(fastaComponent(attributes['species'])) + elif components: + components[-1].extend(line) diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py index 942eac7d16c..823f185535d 100644 --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -886,7 +886,10 @@ class JobHandlerStopQueue(Monitors): .filter((model.Job.state == model.Job.states.DELETED_NEW) & (model.Job.handler == self.app.config.server_name)).all() for job in newly_deleted_jobs: - jobs_to_check.append((job, job.stderr)) + # job.stderr is always a string (job.job_stderr + job.tool_stderr, possibly `''`), + # while any `not None` message returned in self.queue.get_nowait() is interpreted + # as an error, so here we use None if job.stderr is false-y + jobs_to_check.append((job, job.stderr or None)) # Also pull from the queue (in the case of Administrative stopped jobs) try: while 1: diff --git a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh index 81107c568b5..a588fb12b5a 100644 --- a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh +++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh @@ -4,6 +4,7 @@ $headers _galaxy_setup_environment() { local _use_framework_galaxy="$1" + _GALAXY_JOB_DIR="$working_directory" _GALAXY_JOB_HOME_DIR="$working_directory/home" _GALAXY_JOB_TMP_DIR=$tmp_dir_creation_statement $env_setup_commands diff --git a/lib/galaxy/jobs/runners/util/pykube_util.py b/lib/galaxy/jobs/runners/util/pykube_util.py index a86601c324f..0203b9f5046 100644 --- a/lib/galaxy/jobs/runners/util/pykube_util.py +++ b/lib/galaxy/jobs/runners/util/pykube_util.py @@ -47,7 +47,7 @@ def produce_unique_k8s_job_name(app_prefix=None, instance_id=None, job_id=None): if app_prefix: job_name += "%s-" % app_prefix - if instance_id and instance_id > 0: + if instance_id and len(instance_id) > 0: job_name += "%s-" % instance_id return job_name + job_id diff --git a/lib/galaxy/managers/configuration.py b/lib/galaxy/managers/configuration.py index 5da211a6de2..26d5a0c9a04 100644 --- a/lib/galaxy/managers/configuration.py +++ b/lib/galaxy/managers/configuration.py @@ -52,7 +52,6 @@ class ConfigSerializer(base.ModelSerializer): 'lims_doc_url' : _defaults_to("https://usegalaxy.org/u/rkchak/p/sts"), 'default_locale' : _defaults_to(self.app.config.default_locale), 'enable_openid' : _defaults_to(False), - 'enable_beta_ts_api_install' : _defaults_to(False), 'enable_communication_server' : _defaults_to(False), 'communication_server_port' : _defaults_to(None), 'communication_server_host' : _defaults_to(None), diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index 3940e20c4eb..ad2d1bacfeb 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -62,6 +62,7 @@ class JobManager(object): for data_assoc in job.output_datasets: if not self.dataset_manager.is_accessible(data_assoc.dataset.dataset, trans.user): raise ItemAccessibilityException("You are not allowed to rerun this job.") + trans.sa_session.refresh(job) return job diff --git a/lib/galaxy/managers/library_datasets.py b/lib/galaxy/managers/library_datasets.py index 6cdf791fd3a..20b76a1bf50 100644 --- a/lib/galaxy/managers/library_datasets.py +++ b/lib/galaxy/managers/library_datasets.py @@ -101,6 +101,7 @@ class LibraryDatasetsManager(datasets.DatasetAssociationManager): ldda.dbkey = new_genome_build changed = True if changed: + ldda.update_parent_folder_update_times() trans.sa_session.add(ldda) trans.sa_session.flush() return changed @@ -223,6 +224,7 @@ class LibraryDatasetsManager(datasets.DatasetAssociationManager): rval['full_path'] = full_path rval['file_size'] = util.nice_size(int(ldda.get_size())) rval['date_uploaded'] = ldda.create_time.strftime("%Y-%m-%d %I:%M %p") + rval['update_time'] = ldda.update_time.strftime("%Y-%m-%d %I:%M %p") rval['can_user_modify'] = trans.user_is_admin or trans.app.security_agent.can_modify_library_item(current_user_roles, ld) rval['is_unrestricted'] = trans.app.security_agent.dataset_is_public(ldda.dataset) rval['tags'] = self.tag_handler.get_tags_str(ldda.tags) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 1151f94f4b9..bfecde4b327 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -31,6 +31,7 @@ from sqlalchemy import ( not_, or_, select, + text, true, type_coerce, types) @@ -322,27 +323,28 @@ class JobLike(object): return "%s[%s,tool_id=%s]" % (self.__class__.__name__, extra, self.tool_id) - def get_stdout(self): - stdout = self.tool_stdout + @property + def stdout(self): + stdout = self.tool_stdout or '' if self.job_stdout: stdout += "\n" + self.job_stdout return stdout - def set_stdout(self, stdout): + @stdout.setter + def stdout(self, stdout): raise NotImplementedError("Attempt to set stdout, must set tool_stdout or job_stdout") - def get_stderr(self): - stderr = self.tool_stderr + @property + def stderr(self): + stderr = self.tool_stderr or '' if self.job_stderr: stderr += "\n" + self.job_stderr return stderr - def set_stderr(self, stderr): + @stderr.setter + def stderr(self, stderr): raise NotImplementedError("Attempt to set stdout, must set tool_stderr or job_stderr") - stdout = property(get_stdout, set_stdout) - stderr = property(get_stderr, set_stderr) - class User(Dictifiable, RepresentById): use_pbkdf2 = True @@ -3468,6 +3470,7 @@ class LibraryDataset(RepresentById): uploaded_by=ldda.user.email, message=ldda.message, date_uploaded=ldda.create_time.isoformat(), + update_time=ldda.update_time.isoformat(), file_size=int(ldda.get_size()), file_ext=ldda.ext, data_type=ldda.datatype.__class__.__module__ + '.' + ldda.datatype.__class__.__name__, @@ -3636,6 +3639,32 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById): rval['metadata_' + name] = val return rval + def update_parent_folder_update_times(self): + # sets the update_time for all continaing folders up the tree + ldda = self + + sql = text( + ''' + WITH RECURSIVE parent_folders_of(folder_id) AS + (SELECT folder_id + FROM library_dataset + WHERE id = :library_dataset_id + UNION ALL + SELECT library_folder.parent_id + FROM library_folder, parent_folders_of + WHERE library_folder.id = parent_folders_of.folder_id ) + UPDATE library_folder + SET update_time = + (SELECT update_time + FROM library_dataset_dataset_association + WHERE id = :ldda_id) + WHERE exists (SELECT 1 FROM parent_folders_of + WHERE library_folder.id = parent_folders_of.folder_id) + ''').execution_options(autocommit=True) + ret = object_session(self).execute(sql, {'library_dataset_id': ldda.library_dataset_id, 'ldda_id': ldda.id}) + if ret.rowcount < 1: + log.warn('Attempt to updated parent folder times failed: {0} records updated.'.format(ret.rowcount)) + class ExtendedMetadata(RepresentById): def __init__(self, data): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 90caf7f413f..edd12486dab 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -2811,6 +2811,7 @@ def init(file_path, url, engine_options=None, create_tables=False, map_install_m if map_install_models: import galaxy.model.tool_shed_install.mapping # noqa: F401 from galaxy.model import tool_shed_install + galaxy.model.tool_shed_install.mapping.init(url=url, engine_options=engine_options, create_tables=create_tables) model_modules.append(tool_shed_install) result = ModelMapping(model_modules, engine=engine) diff --git a/lib/galaxy/model/migrate/check.py b/lib/galaxy/model/migrate/check.py index 3fe5f987446..07849a64a7f 100644 --- a/lib/galaxy/model/migrate/check.py +++ b/lib/galaxy/model/migrate/check.py @@ -11,6 +11,8 @@ from sqlalchemy import ( from sqlalchemy.exc import NoSuchTableError from sqlalchemy_utils import create_database, database_exists +from galaxy.model import mapping + log = logging.getLogger(__name__) # path relative to galaxy @@ -18,7 +20,7 @@ migrate_repository_directory = os.path.abspath(os.path.dirname(__file__)).replac migrate_repository = repository.Repository(migrate_repository_directory) -def create_or_verify_database(url, galaxy_config_file, engine_options={}, app=None): +def create_or_verify_database(url, galaxy_config_file, engine_options={}, app=None, map_install_models=False): """ Check that the database is use-able, possibly creating it if empty (this is the only time we automatically create tables, otherwise we force the @@ -60,7 +62,16 @@ def create_or_verify_database(url, galaxy_config_file, engine_options={}, app=No migrate_to_current_version(engine, db_schema) meta = MetaData(bind=engine) - if new_database or (app and getattr(app.config, 'database_auto_migrate', False)): + if new_database and app: + log.info("Creating new database from scratch, skipping migrations") + current_version = migrate_repository.version().version + mapping.init(file_path=app.config.file_path, url=url, map_install_models=map_install_models, create_tables=True) + schema.ControlledSchema.create(engine, migrate_repository, version=current_version) + db_schema = schema.ControlledSchema(engine, migrate_repository) + assert db_schema.version == current_version + migrate() + return + elif getattr(app.config, 'database_auto_migrate', False): migrate() return diff --git a/lib/galaxy/model/migrate/versions/0145_add_workflow_step_input.py b/lib/galaxy/model/migrate/versions/0145_add_workflow_step_input.py index 4944fc3ec9c..a5e799104d5 100644 --- a/lib/galaxy/model/migrate/versions/0145_add_workflow_step_input.py +++ b/lib/galaxy/model/migrate/versions/0145_add_workflow_step_input.py @@ -45,7 +45,10 @@ def upgrade(migrate_engine): OldWorkflowStepConnection_table = Table("workflow_step_connection", metadata, autoload=True) for index in OldWorkflowStepConnection_table.indexes: - index.drop() + try: + index.drop() + except Exception: + log.exception("Dropping index '%s' from table '%s' failed", index, OldWorkflowStepConnection_table) OldWorkflowStepConnection_table.rename("workflow_step_connection_preupgrade145") # Try to deregister that table to work around some caching problems it seems. OldWorkflowStepConnection_table.deregister() diff --git a/lib/galaxy/model/orm/scripts.py b/lib/galaxy/model/orm/scripts.py index ddd62fe5700..8cf9292b8b2 100644 --- a/lib/galaxy/model/orm/scripts.py +++ b/lib/galaxy/model/orm/scripts.py @@ -3,15 +3,19 @@ Code to support database helper scripts (create_db.py, manage_db.py, etc...). """ import argparse import logging -import os.path +import os +import sys + +from migrate.versioning.shell import main as migrate_main from galaxy.util.path import get_ext -from galaxy.util.properties import find_config_file, load_app_properties +from galaxy.util.properties import find_config_file, get_data_dir, load_app_properties from galaxy.util.script import populate_config_args log = logging.getLogger(__name__) +DEFAULT_CONFIG_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'config', 'sample')) DEFAULT_CONFIG_NAMES = ['galaxy', 'universe_wsgi'] DEFAULT_CONFIG_PREFIX = '' DEFAULT_DATABASE = 'galaxy' @@ -19,29 +23,29 @@ DEFAULT_DATABASE = 'galaxy' DATABASE = { "galaxy": { - 'repo': 'lib/galaxy/model/migrate', - 'default_sqlite_file': './database/universe.sqlite', + 'repo': 'galaxy/model/migrate', + 'default_sqlite_file': 'universe.sqlite', 'config_override': 'GALAXY_CONFIG_', }, "tools": { - 'repo': 'lib/tool_shed/galaxy_install/migrate', - 'default_sqlite_file': './database/universe.sqlite', + 'repo': 'tool_shed/galaxy_install/migrate', + 'default_sqlite_file': 'universe.sqlite', 'config_override': 'GALAXY_CONFIG_', }, "tool_shed": { - 'repo': 'lib/galaxy/webapps/tool_shed/model/migrate', + 'repo': 'galaxy/webapps/tool_shed/model/migrate', 'config_names': ['tool_shed', 'tool_shed_wsgi'], - 'default_sqlite_file': './database/community.sqlite', + 'default_sqlite_file': 'community.sqlite', 'config_override': 'TOOL_SHED_CONFIG_', 'config_section': 'tool_shed', }, "install": { - 'repo': 'lib/galaxy/model/tool_shed_install/migrate', + 'repo': 'galaxy/model/tool_shed_install/migrate', 'config_prefix': 'install_', - 'default_sqlite_file': './database/install.sqlite', + 'default_sqlite_file': 'install.sqlite', 'config_override': 'GALAXY_INSTALL_CONFIG_', }, } @@ -93,17 +97,17 @@ def get_config(argv, use_argparse=True, cwd=None): ... with open(os.path.join(config_dir, 'config', path), 'w') as f: p.write(f) >>> write_ini('tool_shed.ini', 'database_connection', 'sqlite:///pg/testdb1') >>> config = get_config(['manage_db.py', 'tool_shed'], cwd=config_dir) - >>> config['repo'] - 'lib/galaxy/webapps/tool_shed/model/migrate' + >>> config['repo'].endswith('galaxy/webapps/tool_shed/model/migrate') + True >>> config['db_url'] 'sqlite:///pg/testdb1' - >>> write_ini('galaxy.ini', 'database_file', 'moo.sqlite') + >>> write_ini('galaxy.ini', 'data_dir', '/moo') >>> config = get_config(['manage_db.py'], cwd=config_dir) - >>> uri_with_env = os.getenv("GALAXY_TEST_DBURI", "sqlite:///moo.sqlite?isolation_level=IMMEDIATE") + >>> uri_with_env = os.getenv("GALAXY_TEST_DBURI", "sqlite:////moo/universe.sqlite?isolation_level=IMMEDIATE") >>> config['db_url'] == uri_with_env True - >>> config['repo'] - 'lib/galaxy/model/migrate' + >>> config['repo'].endswith('galaxy/model/migrate') + True >>> rmtree(config_dir) """ config_file, config_section, database = _read_model_arguments(argv, use_argparse=use_argparse) @@ -112,9 +116,11 @@ def get_config(argv, use_argparse=True, cwd=None): config_names = database_defaults.get('config_names', DEFAULT_CONFIG_NAMES) if cwd: cwd = [cwd, os.path.join(cwd, 'config')] + else: + cwd = [DEFAULT_CONFIG_DIR] config_file = find_config_file(config_names, dirs=cwd) - repo = database_defaults['repo'] + repo = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, database_defaults['repo']) config_prefix = database_defaults.get('config_prefix', DEFAULT_CONFIG_PREFIX) config_override = database_defaults.get('config_override', 'GALAXY_CONFIG_') default_sqlite_file = database_defaults['default_sqlite_file'] @@ -128,10 +134,13 @@ def get_config(argv, use_argparse=True, cwd=None): if ("%sdatabase_connection" % config_prefix) in properties: db_url = properties["%sdatabase_connection" % config_prefix] - elif ("%sdatabase_file" % config_prefix) in properties: - database_file = properties["%sdatabase_file" % config_prefix] - db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % database_file else: - db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % default_sqlite_file + db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % os.path.join(get_data_dir(properties), default_sqlite_file) return dict(db_url=db_url, repo=repo, config_file=config_file, database=database) + + +def manage_db(): + # Migrate has its own args, so cannot use argparse + config = get_config(sys.argv, use_argparse=False, cwd=os.getcwd()) + migrate_main(repository=config['repo'], url=config['db_url']) diff --git a/lib/galaxy/model/tool_shed_install/__init__.py b/lib/galaxy/model/tool_shed_install/__init__.py index 430642f60dc..1a7d611967f 100644 --- a/lib/galaxy/model/tool_shed_install/__init__.py +++ b/lib/galaxy/model/tool_shed_install/__init__.py @@ -4,7 +4,7 @@ import os from galaxy.util import asbool from galaxy.util.bunch import Bunch from galaxy.util.dictifiable import Dictifiable -from tool_shed.util import common_util +from galaxy.util.tool_shed import common_util log = logging.getLogger(__name__) diff --git a/lib/galaxy/model/tool_shed_install/migrate/check.py b/lib/galaxy/model/tool_shed_install/migrate/check.py index 7773146965e..a7fffd7381a 100644 --- a/lib/galaxy/model/tool_shed_install/migrate/check.py +++ b/lib/galaxy/model/tool_shed_install/migrate/check.py @@ -13,6 +13,9 @@ from sqlalchemy import ( ) from sqlalchemy.exc import NoSuchTableError +from galaxy.model.tool_shed_install import mapping + + log = logging.getLogger(__name__) # path relative to galaxy @@ -48,6 +51,12 @@ def create_or_verify_database(url, engine_options={}, app=None): # No table means a completely uninitialized database. If we # have an app, we'll set its new_installation setting to True # so the tool migration process will be skipped. + log.info("Creating install database from scratch, skipping migrations") + mapping.init(url=url, create_tables=True) + current_version = migrate_repository.version().version + schema.ControlledSchema.create(engine, migrate_repository, version=current_version) + db_schema = schema.ControlledSchema(engine, migrate_repository) + assert db_schema.version == current_version migrate() return diff --git a/lib/galaxy/tool_util/deps/resolvers/__init__.py b/lib/galaxy/tool_util/deps/resolvers/__init__.py index 7fd9099acde..96369c7d96c 100644 --- a/lib/galaxy/tool_util/deps/resolvers/__init__.py +++ b/lib/galaxy/tool_util/deps/resolvers/__init__.py @@ -1,4 +1,5 @@ """The module defines the abstract interface for dealing tool dependency resolution plugins.""" +import errno from abc import ( ABCMeta, abstractmethod, @@ -95,8 +96,13 @@ class MappableDependencyResolver(object): @staticmethod def _mapping_file_to_list(mapping_file): - with open(mapping_file, "r") as f: - raw_mapping = yaml.safe_load(f) or [] + raw_mapping = [] + try: + with open(mapping_file, "r") as f: + raw_mapping = yaml.safe_load(f) + except (OSError, IOError) as exc: + if exc.errno != errno.ENOENT: + raise return map(RequirementMapping.from_dict, raw_mapping) def _expand_mappings(self, requirement): diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 7379cce952a..c46b524bdec 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -4649,7 +4649,7 @@ define. - Whether to strip whitespace. + Whether to strip leading and trailing whitespace from the calculated value before exporting the environment variable. diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 79589818155..9a07aeed666 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -235,6 +235,11 @@ class ToolBox(BaseGalaxyToolBox): def __init__(self, config_filenames, tool_root_dir, app): self._reload_count = 0 self.tool_location_fetcher = ToolLocationFetcher() + # This is here to deal with the old default value, which doesn't make + # sense in an "installed Galaxy" world. + # FIXME: ./ + if tool_root_dir == './tools': + tool_root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'bundled')) super(ToolBox, self).__init__( config_filenames=config_filenames, tool_root_dir=tool_root_dir, diff --git a/lib/galaxy/tools/bundled b/lib/galaxy/tools/bundled new file mode 120000 index 00000000000..c92f4172dfd --- /dev/null +++ b/lib/galaxy/tools/bundled @@ -0,0 +1 @@ +../../../tools \ No newline at end of file diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py index 1c8b036eb23..4fc77b3e948 100644 --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -6,6 +6,7 @@ users to configure data tables for a local Galaxy instance without needing to modify the tool configurations. """ +import errno import hashlib import logging import os @@ -28,6 +29,11 @@ log = logging.getLogger(__name__) DEFAULT_TABLE_TYPE = 'tabular' +TOOL_DATA_TABLE_CONF_XML = """ + + +""" + class ToolDataPathFiles(object): @@ -61,7 +67,7 @@ class ToolDataPathFiles(object): class ToolDataTableManager(object): """Manages a collection of tool data tables""" - def __init__(self, tool_data_path, config_filename=None): + def __init__(self, tool_data_path, config_filename=None, tool_data_table_config_path_set=None): self.tool_data_path = tool_data_path # This stores all defined data table entries from both the tool_data_table_conf.xml file and the shed_tool_data_table_conf.xml file # at server startup. If tool shed repositories are installed that contain a valid file named tool_data_table_conf.xml.sample, entries @@ -173,7 +179,15 @@ class ToolDataTableManager(object): full_path = os.path.abspath(shed_tool_data_table_config) # FIXME: we should lock changing this file by other threads / head nodes try: - tree = util.parse_xml(full_path) + try: + tree = util.parse_xml(full_path) + except (OSError, IOError) as e: + if e.errno == errno.ENOENT: + with open(full_path, 'w') as fh: + fh.write(TOOL_DATA_TABLE_CONF_XML) + tree = util.parse_xml(full_path) + else: + raise root = tree.getroot() out_elems = [elem for elem in root] except Exception as e: @@ -390,6 +404,12 @@ class TabularToolDataTable(ToolDataTable, Dictifiable): self._update_version() else: self.missing_index_file = filename + # TODO: some data tables need to exist (even if they are empty) + # for tools to load. In an installed Galaxy environment and the + # default tool_data_table_conf.xml, this will emit spurious + # warnings about missing location files that would otherwise be + # empty and we don't care about unless the admin chooses to + # populate them. log.warning("Cannot find index file '%s' for tool data table '%s'" % (filename, self.name)) if filename not in self.filenames or not self.filenames[filename]['found']: diff --git a/lib/galaxy/tools/data_manager/manager.py b/lib/galaxy/tools/data_manager/manager.py index d46418ae23f..9e6f2949203 100644 --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -37,11 +37,19 @@ class DataManagers(object): continue self.load_from_xml(filename) if self.app.config.shed_data_manager_config_file: - self.load_from_xml(self.app.config.shed_data_manager_config_file, store_tool_path=True) + try: + self.load_from_xml(self.app.config.shed_data_manager_config_file, store_tool_path=True) + except (OSError, IOError) as exc: + if exc.errno != errno.ENOENT or self.app.config.shed_data_manager_config_file_set: + raise def load_from_xml(self, xml_filename, store_tool_path=True): try: tree = util.parse_xml(xml_filename) + except (IOError, OSError) as e: + if e.errno != errno.ENOENT or self.app.config.data_manager_config_file_set: + raise + return # default config option and it doesn't exist, which is fine except Exception as e: log.error('There was an error parsing your Data Manager config file "%s": %s' % (xml_filename, e)) return # we are not able to load any data managers diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index 3a30d6372df..2fff4edd3f2 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -547,7 +547,8 @@ class ToolEvaluator(object): os.close(fd) self.__write_workdir_file(config_filename, environment_variable_template, param_dict, strip=environment_variable_def.get("strip", False)) config_file_basename = os.path.basename(config_filename) - environment_variable["value"] = "`cat %s`" % config_file_basename + # environment setup in job file template happens before `cd $working_directory` + environment_variable["value"] = '`cat "$_GALAXY_JOB_DIR/%s"`' % config_file_basename environment_variable["raw"] = True environment_variables.append(environment_variable) diff --git a/lib/galaxy/tools/special_tools.py b/lib/galaxy/tools/special_tools.py index 129b7064a94..5665118bce2 100644 --- a/lib/galaxy/tools/special_tools.py +++ b/lib/galaxy/tools/special_tools.py @@ -1,14 +1,16 @@ import logging +import os + log = logging.getLogger(__name__) SPECIAL_TOOLS = { - "history export": "galaxy/tools/imp_exp/exp_history_to_archive.xml", - "history import": "galaxy/tools/imp_exp/imp_history_from_archive.xml", - "data fetch": "galaxy/tools/data_fetch.xml", + "history export": "imp_exp/exp_history_to_archive.xml", + "history import": "imp_exp/imp_history_from_archive.xml", + "data fetch": "data_fetch.xml", } def load_lib_tools(toolbox): for name, path in SPECIAL_TOOLS.items(): - tool = toolbox.load_hidden_lib_tool(path) + tool = toolbox.load_hidden_lib_tool(os.path.abspath(os.path.join(os.path.dirname(__file__), path))) log.debug("Loaded %s tool: %s", name, tool.id) diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index de8c4bc51a3..aaa01dc489a 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -1,3 +1,4 @@ +import errno import logging import os import string @@ -37,6 +38,11 @@ from .tags import tool_tag_manager log = logging.getLogger(__name__) +SHED_TOOL_CONF_XML = """ + + +""" + # A fake ToolShedRepository constructed from a shed tool conf ToolConfRepository = namedtuple( 'ToolConfRepository', @@ -154,7 +160,21 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): """ log.info("Parsing the tool configuration %s" % config_filename) - tool_conf_source = get_toolbox_parser(config_filename) + try: + tool_conf_source = get_toolbox_parser(config_filename) + except (OSError, IOError) as exc: + for opt in ('shed_tool_conf', 'migrated_tools_config'): + if (config_filename == getattr(self.app.config, opt) and not + getattr(self.app.config, opt + '_set') and + exc.errno == errno.ENOENT): + log.debug("Skipping loading missing default config file: %s", config_filename) + stcd = dict(config_filename=config_filename, + tool_path=self.app.config.shed_tools_dir, + config_elems=[], + create=SHED_TOOL_CONF_XML.format(shed_tools_dir=self.app.config.shed_tools_dir)) + self._dynamic_tool_confs.append(stcd) + return + raise tool_path = tool_conf_source.parse_tool_path() parsing_shed_tool_conf = tool_conf_source.is_shed_tool_conf() if parsing_shed_tool_conf: @@ -1138,7 +1158,9 @@ class BaseGalaxyToolBox(AbstractToolBox): def _init_dependency_manager(self): app_config_dict = self.app.config.config_dict conf_file = app_config_dict.get("dependency_resolvers_config_file") - self.dependency_manager = build_dependency_manager(app_config_dict=app_config_dict, conf_file=conf_file, default_tool_dependency_dir="database/dependencies") + default_tool_dependency_dir = os.path.join(self.app.config.data_dir, "dependencies") + self.dependency_manager = build_dependency_manager(app_config_dict=app_config_dict, conf_file=conf_file, + default_tool_dependency_dir=default_tool_dependency_dir) def reload_dependency_manager(self): self._init_dependency_manager() diff --git a/lib/galaxy/tools/util/maf_utilities.py b/lib/galaxy/tools/util/maf_utilities.py deleted file mode 100644 index 3a5aa531bb2..00000000000 --- a/lib/galaxy/tools/util/maf_utilities.py +++ /dev/null @@ -1,760 +0,0 @@ -#!/usr/bin/env python -""" -Provides wrappers and utilities for working with MAF files and alignments. -""" -# Dan Blankenberg -from __future__ import print_function - -import functools -import logging -import os -import resource -import sys -import tempfile -from copy import deepcopy -from errno import EMFILE - -import bx.align.maf -import bx.interval_index_file -import bx.intervals -from six.moves import xrange - -try: - from string import maketrans -except ImportError: - maketrans = str.maketrans - -assert sys.version_info[:2] >= (2, 6) - -log = logging.getLogger(__name__) - -GAP_CHARS = ['-'] -SRC_SPLIT_CHAR = '.' - - -def src_split(src): - fields = src.split(SRC_SPLIT_CHAR, 1) - spec = fields.pop(0) - if fields: - chrom = fields.pop(0) - else: - chrom = spec - return spec, chrom - - -def src_merge(spec, chrom, contig=None): - if None in [spec, chrom]: - spec = chrom = spec or chrom - return bx.align.maf.src_merge(spec, chrom, contig) - - -def get_species_in_block(block): - species = [] - for c in block.components: - spec, chrom = src_split(c.src) - if spec not in species: - species.append(spec) - return species - - -def tool_fail(msg="Unknown Error"): - print("Fatal Error: %s" % msg, file=sys.stderr) - sys.exit() - - -class TempFileHandler(object): - ''' - Handles creating, opening, closing, and deleting of Temp files, with a - maximum number of files open at one time. - ''' - - DEFAULT_MAX_OPEN_FILES = max(resource.getrlimit(resource.RLIMIT_NOFILE)[0] / 2, 1) - - def __init__(self, max_open_files=None, **kwds): - if max_open_files is None: - max_open_files = self.DEFAULT_MAX_OPEN_FILES - self.max_open_files = max_open_files - self.files = [] - self.open_file_indexes = [] - self.kwds = kwds - - def get_open_tempfile(self, index=None, **kwds): - if index is not None and index in self.open_file_indexes: - self.open_file_indexes.remove(index) - else: - if self.max_open_files: - while len(self.open_file_indexes) >= self.max_open_files: - self.close(self.open_file_indexes[0]) - if index is None: - index = len(self.files) - temp_kwds = dict(self.kwds) - temp_kwds.update(kwds) - # Being able to use delete=True here, would simplify a bit, - # but we support python2.4 in these tools - while True: - try: - tmp_file = tempfile.NamedTemporaryFile(**temp_kwds) - filename = tmp_file.name - break - except OSError as e: - if self.open_file_indexes and e.errno == EMFILE: - self.max_open_files = len(self.open_file_indexes) - self.close(self.open_file_indexes[0]) - else: - raise e - tmp_file.close() - self.files.append(open(filename, 'w+b')) - else: - while True: - try: - self.files[index] = open(self.files[index].name, 'r+b') - break - except OSError as e: - if self.open_file_indexes and e.errno == EMFILE: - self.max_open_files = len(self.open_file_indexes) - self.close(self.open_file_indexes[0]) - else: - raise e - self.files[index].seek(0, 2) - self.open_file_indexes.append(index) - return index, self.files[index] - - def close(self, index, delete=False): - if index in self.open_file_indexes: - self.open_file_indexes.remove(index) - rval = self.files[index].close() - if delete: - try: - os.unlink(self.files[index].name) - except OSError: - pass - return rval - - def flush(self, index): - if index in self.open_file_indexes: - self.files[index].flush() - - def __del__(self): - for i in xrange(len(self.files)): - self.close(i, delete=True) - - -# an object corresponding to a reference layered alignment -class RegionAlignment(object): - - DNA_COMPLEMENT = maketrans("ACGTacgt", "TGCAtgca") - MAX_SEQUENCE_SIZE = sys.maxsize # Maximum length of sequence allowed - - def __init__(self, size, species=[], temp_file_handler=None): - assert size <= self.MAX_SEQUENCE_SIZE, "Maximum length allowed for an individual sequence has been exceeded (%i > %i)." % (size, self.MAX_SEQUENCE_SIZE) - self.size = size - if not temp_file_handler: - temp_file_handler = TempFileHandler() - self.temp_file_handler = temp_file_handler - self.sequences = {} - if not isinstance(species, list): - species = [species] - for spec in species: - self.add_species(spec) - - # add a species to the alignment - def add_species(self, species): - # make temporary sequence files - file_index, fh = self.temp_file_handler.get_open_tempfile() - self.sequences[species] = file_index - fh.write("-" * self.size) - - # returns the names for species found in alignment, skipping names as requested - def get_species_names(self, skip=[]): - if not isinstance(skip, list): - skip = [skip] - names = list(self.sequences.keys()) - for name in skip: - try: - names.remove(name) - except ValueError: - pass - return names - - # returns the sequence for a species - def get_sequence(self, species): - file_index, fh = self.temp_file_handler.get_open_tempfile(self.sequences[species]) - fh.seek(0) - return fh.read() - - # returns the reverse complement of the sequence for a species - def get_sequence_reverse_complement(self, species): - complement = [base for base in self.get_sequence(species).translate(self.DNA_COMPLEMENT)] - complement.reverse() - return "".join(complement) - - # sets a position for a species - def set_position(self, index, species, base): - if len(base) != 1: - raise Exception("A genomic position can only have a length of 1.") - return self.set_range(index, species, base) - # sets a range for a species - - def set_range(self, index, species, bases): - if index >= self.size or index < 0: - raise Exception("Your index (%i) is out of range (0 - %i)." % (index, self.size - 1)) - if len(bases) == 0: - raise Exception("A set of genomic positions can only have a positive length.") - if species not in self.sequences.keys(): - self.add_species(species) - file_index, fh = self.temp_file_handler.get_open_tempfile(self.sequences[species]) - fh.seek(index) - fh.write(bases) - - # Flush temp file of specified species, or all species - def flush(self, species=None): - if species is None: - species = self.sequences.keys() - elif not isinstance(species, list): - species = [species] - for spec in species: - self.temp_file_handler.flush(self.sequences[spec]) - - -class GenomicRegionAlignment(RegionAlignment): - - def __init__(self, start, end, species=[], temp_file_handler=None): - RegionAlignment.__init__(self, end - start, species, temp_file_handler=temp_file_handler) - self.start = start - self.end = end - - -class SplicedAlignment(object): - - DNA_COMPLEMENT = maketrans("ACGTacgt", "TGCAtgca") - - def __init__(self, exon_starts, exon_ends, species=[], temp_file_handler=None): - if not isinstance(exon_starts, list): - exon_starts = [exon_starts] - if not isinstance(exon_ends, list): - exon_ends = [exon_ends] - assert len(exon_starts) == len(exon_ends), "The number of starts does not match the number of sizes." - self.exons = [] - if not temp_file_handler: - temp_file_handler = TempFileHandler() - self.temp_file_handler = temp_file_handler - for i in range(len(exon_starts)): - self.exons.append(GenomicRegionAlignment(exon_starts[i], exon_ends[i], species, temp_file_handler=temp_file_handler)) - - # returns the names for species found in alignment, skipping names as requested - def get_species_names(self, skip=[]): - if not isinstance(skip, list): - skip = [skip] - names = [] - for exon in self.exons: - for name in exon.get_species_names(skip=skip): - if name not in names: - names.append(name) - return names - - # returns the sequence for a species - def get_sequence(self, species): - index, fh = self.temp_file_handler.get_open_tempfile() - for exon in self.exons: - if species in exon.get_species_names(): - seq = exon.get_sequence(species) - # we need to refetch fh here, since exon.get_sequence( species ) uses a tempfile - # and if max==1, it would close fh - index, fh = self.temp_file_handler.get_open_tempfile(index) - fh.write(seq) - else: - fh.write("-" * exon.size) - fh.seek(0) - rval = fh.read() - self.temp_file_handler.close(index, delete=True) - return rval - - # returns the reverse complement of the sequence for a species - def get_sequence_reverse_complement(self, species): - complement = [base for base in self.get_sequence(species).translate(self.DNA_COMPLEMENT)] - complement.reverse() - return "".join(complement) - - # Start and end of coding region - @property - def start(self): - return self.exons[0].start - - @property - def end(self): - return self.exons[-1].end - - -# Open a MAF index using a UID -def maf_index_by_uid(maf_uid, index_location_file): - for line in open(index_location_file): - try: - # read each line, if not enough fields, go to next line - if line[0:1] == "#": - continue - fields = line.split('\t') - if maf_uid == fields[1]: - try: - maf_files = fields[4].replace("\n", "").replace("\r", "").split(",") - return bx.align.maf.MultiIndexed(maf_files, keep_open=True, parse_e_rows=False) - except Exception as e: - raise Exception('MAF UID (%s) found, but configuration appears to be malformed: %s' % (maf_uid, e)) - except Exception: - pass - return None - - -# return ( index, temp_index_filename ) for user maf, if available, or build one and return it, return None when no tempfile is created -def open_or_build_maf_index(maf_file, index_filename, species=None): - try: - return (bx.align.maf.Indexed(maf_file, index_filename=index_filename, keep_open=True, parse_e_rows=False), None) - except Exception: - return build_maf_index(maf_file, species=species) - - -def build_maf_index_species_chromosomes(filename, index_species=None): - species = [] - species_chromosomes = {} - indexes = bx.interval_index_file.Indexes() - blocks = 0 - try: - maf_reader = bx.align.maf.Reader(open(filename)) - while True: - pos = maf_reader.file.tell() - block = next(maf_reader) - if block is None: - break - blocks += 1 - for c in block.components: - spec = c.src - chrom = None - if "." in spec: - spec, chrom = spec.split(".", 1) - if spec not in species: - species.append(spec) - species_chromosomes[spec] = [] - if chrom and chrom not in species_chromosomes[spec]: - species_chromosomes[spec].append(chrom) - if index_species is None or spec in index_species: - forward_strand_start = c.forward_strand_start - forward_strand_end = c.forward_strand_end - try: - forward_strand_start = int(forward_strand_start) - forward_strand_end = int(forward_strand_end) - except ValueError: - continue # start and end are not integers, can't add component to index, goto next component - # this likely only occurs when parse_e_rows is True? - # could a species exist as only e rows? should the - if forward_strand_end > forward_strand_start: - # require positive length; i.e. certain lines have start = end = 0 and cannot be indexed - indexes.add(c.src, forward_strand_start, forward_strand_end, pos, max=c.src_size) - except Exception as e: - # most likely a bad MAF - log.debug('Building MAF index on %s failed: %s' % (filename, e)) - return (None, [], {}, 0) - return (indexes, species, species_chromosomes, blocks) - - -# builds and returns ( index, index_filename ) for specified maf_file -def build_maf_index(maf_file, species=None): - indexes, found_species, species_chromosomes, blocks = build_maf_index_species_chromosomes(maf_file, species) - if indexes is not None: - fd, index_filename = tempfile.mkstemp() - out = os.fdopen(fd, 'w') - indexes.write(out) - out.close() - return (bx.align.maf.Indexed(maf_file, index_filename=index_filename, keep_open=True, parse_e_rows=False), index_filename) - return (None, None) - - -def component_overlaps_region(c, region): - if c is None: - return False - start, end = c.get_forward_strand_start(), c.get_forward_strand_end() - if region.start >= end or region.end <= start: - return False - return True - - -def chop_block_by_region(block, src, region, species=None, mincols=0): - # This chopping method was designed to maintain consistency with how start/end padding gaps have been working in Galaxy thus far: - # behavior as seen when forcing blocks to be '+' relative to src sequence (ref) and using block.slice_by_component( ref, slice_start, slice_end ) - # whether-or-not this is the 'correct' behavior is questionable, but this will at least maintain consistency - # comments welcome - slice_start = block.text_size # max for the min() - slice_end = 0 # min for the max() - old_score = block.score # save old score for later use - # We no longer assume only one occurance of src per block, so we need to check them all - for c in iter_components_by_src(block, src): - if component_overlaps_region(c, region): - if c.text is not None: - rev_strand = False - if c.strand == "-": - # We want our coord_to_col coordinates to be returned from positive stranded component - rev_strand = True - c = c.reverse_complement() - start = max(region.start, c.start) - end = min(region.end, c.end) - start = c.coord_to_col(start) - end = c.coord_to_col(end) - if rev_strand: - # need to orient slice coordinates to the original block direction - slice_len = end - start - end = len(c.text) - start - start = end - slice_len - slice_start = min(start, slice_start) - slice_end = max(end, slice_end) - - if slice_start < slice_end: - block = block.slice(slice_start, slice_end) - if block.text_size > mincols: - # restore old score, may not be accurate, but it is better than 0 for everything? - block.score = old_score - if species is not None: - block = block.limit_to_species(species) - block.remove_all_gap_columns() - return block - return None - - -def orient_block_by_region(block, src, region, force_strand=None): - # loop through components matching src, - # make sure each of these components overlap region - # cache strand for each of overlaping regions - # if force_strand / region.strand not in strand cache, reverse complement - # we could have 2 sequences with same src, overlapping region, on different strands, this would cause no reverse_complementing - strands = [c.strand for c in iter_components_by_src(block, src) if component_overlaps_region(c, region)] - if strands and (force_strand is None and region.strand not in strands) or (force_strand is not None and force_strand not in strands): - block = block.reverse_complement() - return block - - -def get_oriented_chopped_blocks_for_region(index, src, region, species=None, mincols=0, force_strand=None): - for block, idx, offset in get_oriented_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols, force_strand): - yield block - - -def get_oriented_chopped_blocks_with_index_offset_for_region(index, src, region, species=None, mincols=0, force_strand=None): - for block, idx, offset in get_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols): - yield orient_block_by_region(block, src, region, force_strand), idx, offset - - -# split a block with multiple occurances of src into one block per src -def iter_blocks_split_by_src(block, src): - for src_c in iter_components_by_src(block, src): - new_block = bx.align.Alignment(score=block.score, attributes=deepcopy(block.attributes)) - new_block.text_size = block.text_size - for c in block.components: - if c == src_c or c.src != src: - new_block.add_component(deepcopy(c)) # components have reference to alignment, don't want to lose reference to original alignment block in original components - yield new_block - - -# split a block into multiple blocks with all combinations of a species appearing only once per block -def iter_blocks_split_by_species(block, species=None): - def __split_components_by_species(components_by_species, new_block): - if components_by_species: - # more species with components to add to this block - components_by_species = deepcopy(components_by_species) - spec_comps = components_by_species.pop(0) - for c in spec_comps: - newer_block = deepcopy(new_block) - newer_block.add_component(deepcopy(c)) - for value in __split_components_by_species(components_by_species, newer_block): - yield value - else: - # no more components to add, yield this block - yield new_block - - # divide components by species - spec_dict = {} - if not species: - species = [] - for c in block.components: - spec, chrom = src_split(c.src) - if spec not in spec_dict: - spec_dict[spec] = [] - species.append(spec) - spec_dict[spec].append(c) - else: - for spec in species: - spec_dict[spec] = [] - for c in iter_components_by_src_start(block, spec): - spec_dict[spec].append(c) - - empty_block = bx.align.Alignment(score=block.score, attributes=deepcopy(block.attributes)) # should we copy attributes? - empty_block.text_size = block.text_size - # call recursive function to split into each combo of spec/blocks - for value in __split_components_by_species(list(spec_dict.values()), empty_block): - sort_block_components_by_block(value, block) # restore original component order - yield value - - -# generator yielding only chopped and valid blocks for a specified region -def get_chopped_blocks_for_region(index, src, region, species=None, mincols=0): - for block, idx, offset in get_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols): - yield block - - -def get_chopped_blocks_with_index_offset_for_region(index, src, region, species=None, mincols=0): - for block, idx, offset in index.get_as_iterator_with_index_and_offset(src, region.start, region.end): - block = chop_block_by_region(block, src, region, species, mincols) - if block is not None: - yield block, idx, offset - - -# returns a filled region alignment for specified regions -def get_region_alignment(index, primary_species, chrom, start, end, strand='+', species=None, mincols=0, overwrite_with_gaps=True, temp_file_handler=None): - if species is not None: - alignment = RegionAlignment(end - start, species, temp_file_handler=temp_file_handler) - else: - alignment = RegionAlignment(end - start, primary_species, temp_file_handler=temp_file_handler) - return fill_region_alignment(alignment, index, primary_species, chrom, start, end, strand, species, mincols, overwrite_with_gaps) - - -# reduces a block to only positions exisiting in the src provided -def reduce_block_by_primary_genome(block, species, chromosome, region_start): - # returns ( startIndex, {species:texts} - # where texts' contents are reduced to only positions existing in the primary genome - src = "%s.%s" % (species, chromosome) - ref = block.get_component_by_src(src) - start_offset = ref.start - region_start - species_texts = {} - for c in block.components: - species_texts[c.src.split('.')[0]] = list(c.text) - # remove locations which are gaps in the primary species, starting from the downstream end - for i in range(len(species_texts[species]) - 1, -1, -1): - if species_texts[species][i] == '-': - for text in species_texts.values(): - text.pop(i) - for spec, text in species_texts.items(): - species_texts[spec] = ''.join(text) - return (start_offset, species_texts) - - -# fills a region alignment -def fill_region_alignment(alignment, index, primary_species, chrom, start, end, strand='+', species=None, mincols=0, overwrite_with_gaps=True): - region = bx.intervals.Interval(start, end) - region.chrom = chrom - region.strand = strand - primary_src = "%s.%s" % (primary_species, chrom) - - # Order blocks overlaping this position by score, lowest first - blocks = [] - for block, idx, offset in index.get_as_iterator_with_index_and_offset(primary_src, start, end): - score = float(block.score) - for i in range(0, len(blocks)): - if score < blocks[i][0]: - blocks.insert(i, (score, idx, offset)) - break - else: - blocks.append((score, idx, offset)) - - # gap_chars_tuple = tuple( GAP_CHARS ) - gap_chars_str = ''.join(GAP_CHARS) - # Loop through ordered blocks and layer by increasing score - for block_dict in blocks: - for block in iter_blocks_split_by_species(block_dict[1].get_at_offset(block_dict[2])): # need to handle each occurance of sequence in block seperately - if component_overlaps_region(block.get_component_by_src(primary_src), region): - block = chop_block_by_region(block, primary_src, region, species, mincols) # chop block - block = orient_block_by_region(block, primary_src, region) # orient block - start_offset, species_texts = reduce_block_by_primary_genome(block, primary_species, chrom, start) - for spec, text in species_texts.items(): - # we should trim gaps from both sides, since these are not positions in this species genome (sequence) - text = text.rstrip(gap_chars_str) - gap_offset = 0 - # while text.startswith( gap_chars_tuple ): - while True in [text.startswith(gap_char) for gap_char in GAP_CHARS]: # python2.4 doesn't accept a tuple for .startswith() - gap_offset += 1 - text = text[1:] - if not text: - break - if text: - if overwrite_with_gaps: - alignment.set_range(start_offset + gap_offset, spec, text) - else: - for i, char in enumerate(text): - if char not in GAP_CHARS: - alignment.set_position(start_offset + gap_offset + i, spec, char) - return alignment - - -# returns a filled spliced region alignment for specified region with start and end lists -def get_spliced_region_alignment(index, primary_species, chrom, starts, ends, strand='+', species=None, mincols=0, overwrite_with_gaps=True, temp_file_handler=None): - # create spliced alignment object - if species is not None: - alignment = SplicedAlignment(starts, ends, species, temp_file_handler=temp_file_handler) - else: - alignment = SplicedAlignment(starts, ends, [primary_species], temp_file_handler=temp_file_handler) - for exon in alignment.exons: - fill_region_alignment(exon, index, primary_species, chrom, exon.start, exon.end, strand, species, mincols, overwrite_with_gaps) - return alignment - - -# loop through string array, only return non-commented lines -def line_enumerator(lines, comment_start='#'): - i = 0 - for line in lines: - if not line.startswith(comment_start): - i += 1 - yield (i, line) - - -# read a GeneBed file, return list of starts, ends, raw fields -def get_starts_ends_fields_from_gene_bed(line): - # Starts and ends for exons - starts = [] - ends = [] - - fields = line.split() - # Requires atleast 12 BED columns - if len(fields) < 12: - raise Exception("Not a proper 12 column BED line (%s)." % line) - tx_start = int(fields[1]) - strand = fields[5] - if strand != '-': - strand = '+' # Default strand is + - cds_start = int(fields[6]) - cds_end = int(fields[7]) - - # Calculate and store starts and ends of coding exons - region_start, region_end = cds_start, cds_end - exon_starts = list(map(int, fields[11].rstrip(',\n').split(','))) - exon_starts = [x + tx_start for x in exon_starts] - exon_ends = list(map(int, fields[10].rstrip(',').split(','))) - exon_ends = [x + y for x, y in zip(exon_starts, exon_ends)] - for start, end in zip(exon_starts, exon_ends): - start = max(start, region_start) - end = min(end, region_end) - if start < end: - starts.append(start) - ends.append(end) - return (starts, ends, fields) - - -def iter_components_by_src(block, src): - for c in block.components: - if c.src == src: - yield c - - -def get_components_by_src(block, src): - return [value for value in iter_components_by_src(block, src)] - - -def iter_components_by_src_start(block, src): - for c in block.components: - if c.src.startswith(src): - yield c - - -def get_components_by_src_start(block, src): - return [value for value in iter_components_by_src_start(block, src)] - - -def sort_block_components_by_block(block1, block2): - # orders the components in block1 by the index of the component in block2 - # block1 must be a subset of block2 - # occurs in-place - return block1.components.sort(key=functools.cmp_to_key(lambda x, y: block2.components.index(x) - block2.components.index(y))) - - -def get_species_in_maf(maf_filename): - species = [] - for block in bx.align.maf.Reader(open(maf_filename)): - for spec in get_species_in_block(block): - if spec not in species: - species.append(spec) - return species - - -def parse_species_option(species): - if species: - species = species.split(',') - if 'None' not in species: - return species - return None # provided species was '', None, or had 'None' in it - - -def remove_temp_index_file(index_filename): - try: - os.unlink(index_filename) - except Exception: - pass - -# Below are methods to deal with FASTA files - - -def get_fasta_header(component, attributes={}, suffix=None): - header = ">%s(%s):%i-%i|" % (component.src, component.strand, component.get_forward_strand_start(), component.get_forward_strand_end()) - for key, value in attributes.items(): - header = "%s%s=%s|" % (header, key, value) - if suffix: - header = "%s%s" % (header, suffix) - else: - header = "%s%s" % (header, src_split(component.src)[0]) - return header - - -def get_attributes_from_fasta_header(header): - if not header: - return {} - attributes = {} - header = header.lstrip('>') - header = header.strip() - fields = header.split('|') - try: - region = fields[0] - region = region.split('(', 1) - temp = region[0].split('.', 1) - attributes['species'] = temp[0] - if len(temp) == 2: - attributes['chrom'] = temp[1] - else: - attributes['chrom'] = temp[0] - region = region[1].split(')', 1) - attributes['strand'] = region[0] - region = region[1].lstrip(':').split('-') - attributes['start'] = int(region[0]) - attributes['end'] = int(region[1]) - except Exception: - # fields 0 is not a region coordinate - pass - if len(fields) > 2: - for i in range(1, len(fields) - 1): - prop = fields[i].split('=', 1) - if len(prop) == 2: - attributes[prop[0]] = prop[1] - if len(fields) > 1: - attributes['__suffix__'] = fields[-1] - return attributes - - -def iter_fasta_alignment(filename): - class fastaComponent(object): - def __init__(self, species, text=""): - self.species = species - self.text = text - - def extend(self, text): - self.text = self.text + text.replace('\n', '').replace('\r', '').strip() - # yields a list of fastaComponents for a FASTA file - f = open(filename, 'rb') - components = [] - # cur_component = None - while True: - line = f.readline() - if not line: - if components: - yield components - return - line = line.strip() - if not line: - if components: - yield components - components = [] - elif line.startswith('>'): - attributes = get_attributes_from_fasta_header(line) - components.append(fastaComponent(attributes['species'])) - elif components: - components[-1].extend(line) diff --git a/lib/galaxy/tools/util/maf_utilities.py b/lib/galaxy/tools/util/maf_utilities.py new file mode 120000 index 00000000000..07bafe2d1a5 --- /dev/null +++ b/lib/galaxy/tools/util/maf_utilities.py @@ -0,0 +1 @@ +../../datatypes/util/maf_utilities.py \ No newline at end of file diff --git a/lib/galaxy/tools/verify/interactor.py b/lib/galaxy/tools/verify/interactor.py index 72ddd5b4ac4..54d1442577d 100644 --- a/lib/galaxy/tools/verify/interactor.py +++ b/lib/galaxy/tools/verify/interactor.py @@ -244,7 +244,7 @@ class GalaxyInteractorApi(object): def wait_for_job(self, job_id, history_id, maxseconds): self.wait_for(lambda: not self.__job_ready(job_id, history_id), maxseconds=maxseconds) - def wait_for(self, func, **kwd): + def wait_for(self, func, what='Tool test run', **kwd): sleep_amount = 0.2 slept = 0 walltime_exceeded = int(kwd.get("maxseconds", DEFAULT_TOOL_TEST_WAIT)) @@ -258,7 +258,7 @@ class GalaxyInteractorApi(object): else: return - message = 'Tool test run exceeded walltime [total %s, max %s], terminating.' % (slept, walltime_exceeded) + message = '%s exceeded walltime [total %s, max %s], terminating.' % (what, slept, walltime_exceeded) log.info(message) raise AssertionError(message) @@ -579,7 +579,7 @@ class GalaxyInteractorApi(object): try: test_user = [user for user in all_users if user["email"] == email][0] except IndexError: - username = re.sub('[^a-z-]', '--', email.lower()) + username = re.sub(r"[^a-z-\d]", '--', email.lower()) password = password or 'testpass' # If remote user middleware is enabled - this endpoint consumes # ``remote_user_email`` otherwise it requires ``email``, ``password`` diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index cf5125a6af9..39362985dd1 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -59,6 +59,11 @@ except ImportError: docutils_core = None docutils_html4css1 = None +try: + import uwsgi +except ImportError: + uwsgi = None + from .inflection import English, Inflector from .logging import get_logger from .path import safe_contains, safe_makedirs, safe_relpath # noqa: F401 @@ -1540,7 +1545,8 @@ def safe_str_cmp(a, b): return rv == 0 -galaxy_root_path = os.path.join(__path__[0], "..", "..", "..") +galaxy_root_path = os.path.join(__path__[0], os.pardir, os.pardir, os.pardir) +galaxy_samples_path = os.path.join(__path__[0], os.pardir, 'config', 'sample') def galaxy_directory(): @@ -1550,6 +1556,10 @@ def galaxy_directory(): return root_path +def galaxy_samples_directory(): + return os.path.abspath(galaxy_samples_path) + + def config_directories_from_setting(directories_setting, galaxy_root=galaxy_root_path): """ Parse the ``directories_setting`` into a list of relative or absolute @@ -1675,6 +1685,29 @@ def download_to_file(url, dest_file_path, timeout=30, chunk_size=2 ** 20): f.write(chunk) +def _executable(): + exe = sys.executable + if exe.endswith('uwsgi'): + virtualenv = None + if uwsgi is not None: + for name in ('home', 'virtualenv', 'venv', 'pyhome'): + if name in uwsgi.opt: + virtualenv = uwsgi.opt[name] + break + if virtualenv is None and 'VIRTUAL_ENV' in os.environ: + virtualenv = os.environ['VIRTUAL_ENV'] + if virtualenv is not None: + exe = os.path.join(virtualenv, 'bin', 'python') + else: + exe = os.path.join(os.path.dirname(exe), 'python') + if not os.path.exists(exe): + exe = 'python' + return exe + + +executable = _executable() + + class ExecutionTimer(object): def __init__(self): diff --git a/lib/galaxy/util/properties.py b/lib/galaxy/util/properties.py index 83dc94ec272..7cc131145ac 100644 --- a/lib/galaxy/util/properties.py +++ b/lib/galaxy/util/properties.py @@ -71,11 +71,13 @@ def load_app_properties( config_prefix="GALAXY_CONFIG_" ): properties = kwds.copy() if kwds else {} + properties['__file__'] = None if config_file is None: config_file = ini_file config_section = ini_section if config_file: + properties['__file__'] = os.path.abspath(config_file) if not has_ext(config_file, 'yaml', aliases=True, ignore='sample'): if config_section is None: config_section = "app:main" @@ -169,6 +171,31 @@ class NicerConfigParser(ConfigParser): raise +def _running_from_source(): + try: + # is there a better way to do this? + assert os.path.exists('run.sh') + assert os.path.exists('lib/galaxy/__init__.py') + assert os.path.exists('scripts/common_startup.sh') + return True + except AssertionError: + return False + + +running_from_source = _running_from_source() + + +def get_data_dir(properties): + data_dir = properties.get('data_dir', None) + if data_dir is None: + if running_from_source: + data_dir = './database' + else: + config_dir = properties.get('config_dir', os.path.dirname(properties['__file__'])) + data_dir = os.path.join(config_dir, 'data') + return data_dir + + def __get_all_configs(dirs, names): return list(filter(os.path.exists, starmap(os.path.join, product(dirs, names)))) @@ -196,4 +223,4 @@ def __default_properties(path): } -__all__ = ('find_config_file', 'load_app_properties', 'NicerConfigParser') +__all__ = ('find_config_file', 'get_data_dir', 'load_app_properties', 'NicerConfigParser', 'running_from_source') diff --git a/lib/galaxy/util/tool_shed/__init__.py b/lib/galaxy/util/tool_shed/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/galaxy/util/tool_shed/common_util.py b/lib/galaxy/util/tool_shed/common_util.py new file mode 100644 index 00000000000..41ac1cf1534 --- /dev/null +++ b/lib/galaxy/util/tool_shed/common_util.py @@ -0,0 +1,377 @@ +import errno +import json +import logging +import os + +from routes import url_for +from six.moves.urllib.parse import urljoin + +from galaxy import util +from galaxy.util.odict import odict +from galaxy.util.tool_shed import encoding_util, xml_util + +log = logging.getLogger(__name__) + +REPOSITORY_OWNER = 'devteam' +TOOL_MIGRATION_SCRIPTS_DIR = os.path.abspath(os.path.join( + os.path.dirname(__file__), os.pardir, 'galaxy_install', 'migrate', 'scripts')) +TOOL_MIGRATION_VERSIONS_DIR = os.path.abspath(os.path.join( + os.path.dirname(__file__), os.pardir, 'galaxy_install', 'migrate', 'versions')) + + +def accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies): + if tool_shed_accessible: + if tool_dependencies: + for tool_dependency in tool_dependencies: + if tool_dependency not in all_tool_dependencies: + all_tool_dependencies.append(tool_dependency) + return all_tool_dependencies + + +def check_for_missing_tools(app, tool_panel_configs, latest_tool_migration_script_number): + # Get the 000x_tools.xml file associated with the current migrate_tools version number. + tools_xml_file_path = os.path.abspath(os.path.join(os.path.dirname(__file__), + os.pardir, 'galaxy_install', + 'migrate', 'scripts', + '%04d_tools.xml' % latest_tool_migration_script_number)) + # Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config. + migrated_tool_configs_dict = odict() + tree, error_message = xml_util.parse_xml(tools_xml_file_path) + if tree is None: + return False, odict() + root = tree.getroot() + tool_shed = root.get('name') + tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, tool_shed) + # The default behavior is that the tool shed is down. + tool_shed_accessible = False + missing_tool_configs_dict = odict() + if tool_shed_url: + for elem in root: + if elem.tag == 'repository': + repository_dependencies = [] + all_tool_dependencies = [] + repository_name = elem.get('name') + changeset_revision = elem.get('changeset_revision') + tool_shed_accessible, repository_dependencies_dict = get_repository_dependencies(app, + tool_shed_url, + repository_name, + REPOSITORY_OWNER, + changeset_revision) + if tool_shed_accessible: + # Accumulate all tool dependencies defined for repository dependencies for display to the user. + for rd_key, rd_tups in repository_dependencies_dict.items(): + if rd_key in ['root_key', 'description']: + continue + for rd_tup in rd_tups: + tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + parse_repository_dependency_tuple(rd_tup) + tool_shed_accessible, tool_dependencies = get_tool_dependencies(app, + tool_shed_url, + name, + owner, + changeset_revision) + all_tool_dependencies = accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies) + tool_shed_accessible, tool_dependencies = get_tool_dependencies(app, + tool_shed_url, + repository_name, + REPOSITORY_OWNER, + changeset_revision) + all_tool_dependencies = accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies) + for tool_elem in elem.findall('tool'): + tool_config_file_name = tool_elem.get('file') + if tool_config_file_name: + # We currently do nothing with repository dependencies except install them (we do not display repositories that will be + # installed to the user). However, we'll store them in the following dictionary in case we choose to display them in the + # future. + dependencies_dict = dict(tool_dependencies=all_tool_dependencies, + repository_dependencies=repository_dependencies) + migrated_tool_configs_dict[tool_config_file_name] = dependencies_dict + else: + break + if tool_shed_accessible: + # Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names. + for tool_panel_config in tool_panel_configs: + tree, error_message = xml_util.parse_xml(tool_panel_config) + if tree: + root = tree.getroot() + for elem in root: + if elem.tag == 'tool': + missing_tool_configs_dict = check_tool_tag_set(elem, migrated_tool_configs_dict, missing_tool_configs_dict) + elif elem.tag == 'section': + for section_elem in elem: + if section_elem.tag == 'tool': + missing_tool_configs_dict = check_tool_tag_set(section_elem, migrated_tool_configs_dict, missing_tool_configs_dict) + else: + exception_msg = '\n\nThe entry for the main Galaxy tool shed at %s is missing from the %s file. ' % (tool_shed, app.config.tool_sheds_config) + exception_msg += 'The entry for this tool shed must always be available in this file, so re-add it before attempting to start your Galaxy server.\n' + raise Exception(exception_msg) + return tool_shed_accessible, missing_tool_configs_dict + + +def check_tool_tag_set(elem, migrated_tool_configs_dict, missing_tool_configs_dict): + file_path = elem.get('file', None) + if file_path: + name = os.path.basename(file_path) + for migrated_tool_config in migrated_tool_configs_dict.keys(): + if migrated_tool_config in [file_path, name]: + missing_tool_configs_dict[name] = migrated_tool_configs_dict[migrated_tool_config] + return missing_tool_configs_dict + + +def generate_clone_url_for_installed_repository(app, repository): + """Generate the URL for cloning a repository that has been installed into a Galaxy instance.""" + tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, str(repository.tool_shed)) + return util.build_url(tool_shed_url, pathspec=['repos', str(repository.owner), str(repository.name)]) + + +def generate_clone_url_for_repository_in_tool_shed(user, repository): + """Generate the URL for cloning a repository that is in the tool shed.""" + base_url = url_for('/', qualified=True).rstrip('/') + if user: + protocol, base = base_url.split('://') + username = '%s@' % user.username + return '%s://%s%s/repos/%s/%s' % (protocol, username, base, repository.user.username, repository.name) + else: + return '%s/repos/%s/%s' % (base_url, repository.user.username, repository.name) + + +def generate_clone_url_from_repo_info_tup(app, repo_info_tup): + """Generate the URL for cloning a repository given a tuple of toolshed, name, owner, changeset_revision.""" + # Example tuple: ['http://localhost:9009', 'blast_datatypes', 'test', '461a4216e8ab', False] + toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + parse_repository_dependency_tuple(repo_info_tup) + tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, toolshed) + # Don't include the changeset_revision in clone urls. + return util.build_url(tool_shed_url, pathspec=['repos', owner, name]) + + +def get_non_shed_tool_panel_configs(app): + """Get the non-shed related tool panel configs - there can be more than one, and the default is tool_conf.xml.""" + config_filenames = [] + for config_filename in app.config.tool_configs: + # Any config file that includes a tool_path attribute in the root tag set like the following is shed-related. + # + try: + tree, error_message = xml_util.parse_xml(config_filename) + except (OSError, IOError) as exc: + if (config_filename == app.config.shed_tool_conf and not + app.config.shed_tool_conf_set and + exc.errno == errno.ENOENT): + continue + raise + if tree is None: + continue + root = tree.getroot() + tool_path = root.get('tool_path', None) + if tool_path is None: + config_filenames.append(config_filename) + return config_filenames + + +def get_repository_dependencies(app, tool_shed_url, repository_name, repository_owner, changeset_revision): + repository_dependencies_dict = {} + tool_shed_accessible = True + params = dict(name=repository_name, owner=repository_owner, changeset_revision=changeset_revision) + pathspec = ['repository', 'get_repository_dependencies'] + try: + raw_text = util.url_get(tool_shed_url, password_mgr=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params) + tool_shed_accessible = True + except Exception as e: + tool_shed_accessible = False + log.warning("The URL\n%s\nraised the exception:\n%s\n", util.build_url(tool_shed_url, pathspec=pathspec, params=params), e) + if tool_shed_accessible: + if len(raw_text) > 2: + encoded_text = json.loads(util.unicodify(raw_text)) + repository_dependencies_dict = encoding_util.tool_shed_decode(encoded_text) + return tool_shed_accessible, repository_dependencies_dict + + +def get_protocol_from_tool_shed_url(tool_shed_url): + """Return the protocol from the received tool_shed_url if it exists.""" + try: + if tool_shed_url.find('://') > 0: + return tool_shed_url.split('://')[0].lower() + except Exception: + # We receive a lot of calls here where the tool_shed_url is None. The container_util uses + # that value when creating a header row. If the tool_shed_url is not None, we have a problem. + if tool_shed_url is not None: + log.exception("Handled exception getting the protocol from Tool Shed URL %s", str(tool_shed_url)) + # Default to HTTP protocol. + return 'http' + + +def get_tool_dependencies(app, tool_shed_url, repository_name, repository_owner, changeset_revision): + tool_dependencies = [] + tool_shed_accessible = True + params = dict(name=repository_name, owner=repository_owner, changeset_revision=changeset_revision) + pathspec = ['repository', 'get_tool_dependencies'] + try: + text = util.url_get(tool_shed_url, password_mgr=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params) + tool_shed_accessible = True + except Exception as e: + tool_shed_accessible = False + log.warning("The URL\n%s\nraised the exception:\n%s\n", util.build_url(tool_shed_url, pathspec=pathspec, params=params), e) + if tool_shed_accessible: + if text: + tool_dependencies_dict = encoding_util.tool_shed_decode(text) + for requirements_dict in tool_dependencies_dict.values(): + tool_dependency_name = requirements_dict['name'] + tool_dependency_version = requirements_dict['version'] + tool_dependency_type = requirements_dict['type'] + tool_dependencies.append((tool_dependency_name, tool_dependency_version, tool_dependency_type)) + return tool_shed_accessible, tool_dependencies + + +def get_tool_shed_repository_ids(as_string=False, **kwd): + tsrid = kwd.get('tool_shed_repository_id', None) + tsridslist = util.listify(kwd.get('tool_shed_repository_ids', None)) + if not tsridslist: + tsridslist = util.listify(kwd.get('id', None)) + if tsridslist is not None: + if tsrid is not None and tsrid not in tsridslist: + tsridslist.append(tsrid) + if as_string: + return ','.join(tsridslist) + return tsridslist + else: + tsridslist = util.listify(kwd.get('ordered_tsr_ids', None)) + if tsridslist is not None: + if as_string: + return ','.join(tsridslist) + return tsridslist + if as_string: + return '' + return [] + + +def get_tool_shed_url_from_tool_shed_registry(app, tool_shed): + """ + The value of tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is + something like: http://toolshed.g2.bx.psu.edu/ + """ + cleaned_tool_shed = remove_protocol_from_tool_shed_url(tool_shed) + for shed_url in app.tool_shed_registry.tool_sheds.values(): + if shed_url.find(cleaned_tool_shed) >= 0: + if shed_url.endswith('/'): + shed_url = shed_url.rstrip('/') + return shed_url + # The tool shed from which the repository was originally installed must no longer be configured in tool_sheds_conf.xml. + return None + + +def get_tool_shed_repository_url(app, tool_shed, owner, name): + tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, tool_shed) + if tool_shed_url: + # Append a slash to the tool shed URL, because urlparse.urljoin will eliminate + # the last part of a URL if it does not end with a forward slash. + tool_shed_url = '%s/' % tool_shed_url + return urljoin(tool_shed_url, 'view/%s/%s' % (owner, name)) + return tool_shed_url + + +def get_user_by_username(app, username): + """Get a user from the database by username.""" + sa_session = app.model.context.current + try: + user = sa_session.query(app.model.User) \ + .filter(app.model.User.table.c.username == username) \ + .one() + return user + except Exception: + return None + + +def handle_galaxy_url(trans, **kwd): + galaxy_url = kwd.get('galaxy_url', None) + if galaxy_url: + trans.set_cookie(galaxy_url, name='toolshedgalaxyurl') + else: + galaxy_url = trans.get_cookie(name='toolshedgalaxyurl') + return galaxy_url + + +def handle_tool_shed_url_protocol(app, shed_url): + """Handle secure and insecure HTTP protocol since they may change over time.""" + try: + if app.name == 'galaxy': + url = remove_protocol_from_tool_shed_url(shed_url) + tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, url) + else: + tool_shed_url = str(url_for('/', qualified=True)).rstrip('/') + return tool_shed_url + except Exception: + # We receive a lot of calls here where the tool_shed_url is None. The container_util uses + # that value when creating a header row. If the tool_shed_url is not None, we have a problem. + if shed_url is not None: + log.exception("Handled exception removing protocol from URL %s", str(shed_url)) + return shed_url + + +def parse_repository_dependency_tuple(repository_dependency_tuple, contains_error=False): + # Default both prior_installation_required and only_if_compiling_contained_td to False in cases where metadata should be reset on the + # repository containing the repository_dependency definition. + prior_installation_required = 'False' + only_if_compiling_contained_td = 'False' + if contains_error: + if len(repository_dependency_tuple) == 5: + tool_shed, name, owner, changeset_revision, error = repository_dependency_tuple + elif len(repository_dependency_tuple) == 6: + tool_shed, name, owner, changeset_revision, prior_installation_required, error = repository_dependency_tuple + elif len(repository_dependency_tuple) == 7: + tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, error = \ + repository_dependency_tuple + return tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, error + else: + if len(repository_dependency_tuple) == 4: + tool_shed, name, owner, changeset_revision = repository_dependency_tuple + elif len(repository_dependency_tuple) == 5: + tool_shed, name, owner, changeset_revision, prior_installation_required = repository_dependency_tuple + elif len(repository_dependency_tuple) == 6: + tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = repository_dependency_tuple + return tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td + + +def remove_port_from_tool_shed_url(tool_shed_url): + """Return a partial Tool Shed URL, eliminating the port if it exists.""" + try: + if tool_shed_url.find(':') > 0: + # Eliminate the port, if any, since it will result in an invalid directory name. + new_tool_shed_url = tool_shed_url.split(':')[0] + else: + new_tool_shed_url = tool_shed_url + return new_tool_shed_url.rstrip('/') + except Exception: + # We receive a lot of calls here where the tool_shed_url is None. The container_util uses + # that value when creating a header row. If the tool_shed_url is not None, we have a problem. + if tool_shed_url is not None: + log.exception("Handled exception removing the port from Tool Shed URL %s", str(tool_shed_url)) + return tool_shed_url + + +def remove_protocol_and_port_from_tool_shed_url(tool_shed_url): + """Return a partial Tool Shed URL, eliminating the protocol and/or port if either exists.""" + tool_shed = remove_protocol_from_tool_shed_url(tool_shed_url) + tool_shed = remove_port_from_tool_shed_url(tool_shed) + return tool_shed + + +def remove_protocol_and_user_from_clone_url(repository_clone_url): + """Return a URL that can be used to clone a repository, eliminating the protocol and user if either exists.""" + if repository_clone_url.find('@') > 0: + # We have an url that includes an authenticated user, something like: + # http://test@bx.psu.edu:9009/repos/some_username/column + items = repository_clone_url.split('@') + tmp_url = items[1] + elif repository_clone_url.find('//') > 0: + # We have an url that includes only a protocol, something like: + # http://bx.psu.edu:9009/repos/some_username/column + items = repository_clone_url.split('//') + tmp_url = items[1] + else: + tmp_url = repository_clone_url + return tmp_url.rstrip('/') + + +def remove_protocol_from_tool_shed_url(tool_shed_url): + """Return a partial Tool Shed URL, eliminating the protocol if it exists.""" + return util.remove_protocol_from_url(tool_shed_url) diff --git a/lib/galaxy/util/tool_shed/encoding_util.py b/lib/galaxy/util/tool_shed/encoding_util.py new file mode 100644 index 00000000000..57c1a11c8a7 --- /dev/null +++ b/lib/galaxy/util/tool_shed/encoding_util.py @@ -0,0 +1,41 @@ +import binascii +import json + +from galaxy.util import ( + smart_str, + unicodify +) +from galaxy.util.hash_util import hmac_new + + +encoding_sep = '__esep__' +encoding_sep2 = '__esepii__' + + +def tool_shed_decode(value): + # Extract and verify hash + value = unicodify(value) + a, b = value.split(":") + value = binascii.unhexlify(b) + test = hmac_new(b'ToolShedAndGalaxyMustHaveThisSameKey', value) + assert a == test + # Restore from string + values = None + value = unicodify(value) + try: + values = json.loads(value) + except Exception: + pass + if values is None: + values = value + return values + + +def tool_shed_encode(val): + if isinstance(val, dict) or isinstance(val, list): + value = json.dumps(val) + else: + value = val + a = hmac_new(b'ToolShedAndGalaxyMustHaveThisSameKey', smart_str(value)) + b = unicodify(binascii.hexlify(smart_str(value))) + return "%s:%s" % (a, b) diff --git a/lib/galaxy/util/tool_shed/xml_util.py b/lib/galaxy/util/tool_shed/xml_util.py new file mode 100644 index 00000000000..5e7e9db05e4 --- /dev/null +++ b/lib/galaxy/util/tool_shed/xml_util.py @@ -0,0 +1,92 @@ +import io +import logging +import os +import tempfile +from xml.etree import ElementTree as XmlET + +from galaxy.util import ( + xml_to_string +) + +log = logging.getLogger(__name__) + + +class Py27CommentedTreeBuilder(XmlET.TreeBuilder): + + def doctype(*args): + # handle deprecation warning for XMLParsing a file with DOCTYPE + pass + + def comment(self, data): + self.start(XmlET.Comment, {}) + self.data(data) + self.end(XmlET.Comment) + + +def create_and_write_tmp_file(elem): + tmp_str = xml_to_string(elem, pretty=True) + with tempfile.NamedTemporaryFile(prefix="tmp-toolshed-cawrf", delete=False) as fh: + tmp_filename = fh.name + with io.open(tmp_filename, mode='w', encoding='utf-8') as fh: + fh.write(tmp_str) + return tmp_filename + + +def create_element(tag, attributes=None, sub_elements=None): + """ + Create a new element whose tag is the value of the received tag, and whose attributes are all + key / value pairs in the received attributes and sub_elements. + """ + if tag: + elem = XmlET.Element(tag) + if attributes: + # The received attributes is an odict to preserve ordering. + for k, v in attributes.items(): + elem.set(k, v) + if sub_elements: + # The received attributes is an odict. These handle information that tends to be + # long text including paragraphs (e.g., description and long_description. + for k, v in sub_elements.items(): + # Don't include fields that are blank. + if v: + if k == 'packages': + # The received sub_elements is an odict whose key is 'packages' and whose + # value is a list of ( name, version ) tuples. + for v_tuple in v: + sub_elem = XmlET.SubElement(elem, 'package') + sub_elem_name, sub_elem_version = v_tuple + sub_elem.set('name', sub_elem_name) + sub_elem.set('version', sub_elem_version) + elif isinstance(v, list): + sub_elem = XmlET.SubElement(elem, k) + # If v is a list, then it must be a list of tuples where the first + # item is the tag and the second item is the text value. + for v_tuple in v: + if len(v_tuple) == 2: + v_tag = v_tuple[0] + v_text = v_tuple[1] + # Don't include fields that are blank. + if v_text: + v_elem = XmlET.SubElement(sub_elem, v_tag) + v_elem.text = v_text + else: + sub_elem = XmlET.SubElement(elem, k) + sub_elem.text = v + return elem + return None + + +def parse_xml(file_name): + """Returns a parsed xml tree with comments intact.""" + error_message = '' + if not os.path.exists(file_name): + return None, "File does not exist %s" % str(file_name) + + with open(file_name, 'r') as fobj: + try: + tree = XmlET.parse(fobj, parser=XmlET.XMLParser(target=Py27CommentedTreeBuilder())) + except Exception as e: + error_message = "Exception attempting to parse %s: %s" % (str(file_name), str(e)) + log.exception(error_message) + return None, error_message + return tree, error_message diff --git a/lib/galaxy/web/framework/static b/lib/galaxy/web/framework/static new file mode 120000 index 00000000000..5feb6c21f47 --- /dev/null +++ b/lib/galaxy/web/framework/static @@ -0,0 +1 @@ +../../../../static \ No newline at end of file diff --git a/lib/galaxy/web/framework/templates b/lib/galaxy/web/framework/templates new file mode 120000 index 00000000000..f8a06d1d1c8 --- /dev/null +++ b/lib/galaxy/web/framework/templates @@ -0,0 +1 @@ +../../../../templates \ No newline at end of file diff --git a/lib/galaxy/web/framework/webapp.py b/lib/galaxy/web/framework/webapp.py index f59ac0b8c6b..569a0c0a3f4 100644 --- a/lib/galaxy/web/framework/webapp.py +++ b/lib/galaxy/web/framework/webapp.py @@ -87,11 +87,16 @@ class WebApplication(base.WebApplication): def create_mako_template_lookup(self, galaxy_app, name): paths = [] + # FIXME: should be os.path.join (galaxy_root, 'templates')? + if galaxy_app.config.template_path == './templates': + template_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'templates')) + else: + template_path = galaxy_app.config.template_path # First look in webapp specific directory if name is not None: - paths.append(os.path.join(galaxy_app.config.template_path, 'webapps', name)) + paths.append(os.path.join(template_path, 'webapps', name)) # Then look in root directory - paths.append(galaxy_app.config.template_path) + paths.append(template_path) # Create TemplateLookup with a small cache return mako.lookup.TemplateLookup(directories=paths, module_directory=galaxy_app.config.template_cache, @@ -962,6 +967,10 @@ class GalaxyWebTransaction(base.DefaultWebTransaction, return str(template) +def default_url_path(path): + return os.path.abspath(os.path.join(os.path.dirname(__file__), path)) + + def build_native_uwsgi_app(paste_factory, config_section): """uwsgi can load paste factories with --ini-paste, but this builds non-paste uwsgi apps. @@ -986,10 +995,13 @@ def build_url_map(app, global_conf, local_conf): # Send to dynamic app by default urlmap["/"] = app # Define static mappings from config - urlmap["/static"] = Static(conf.get("static_dir", "./static/"), cache_time) - urlmap["/images"] = Static(conf.get("static_images_dir", "./static/images"), cache_time) - urlmap["/static/scripts"] = Static(conf.get("static_scripts_dir", "./static/scripts/"), cache_time) - urlmap["/static/style"] = Static(conf.get("static_style_dir", "./static/style/blue"), cache_time) - urlmap["/favicon.ico"] = Static(conf.get("static_favicon_dir", "./static/favicon.ico"), cache_time) - urlmap["/robots.txt"] = Static(conf.get("static_robots_txt", "./static/robots.txt"), cache_time) + urlmap["/static"] = Static(conf.get("static_dir", default_url_path("static/")), cache_time) + urlmap["/images"] = Static(conf.get("static_images_dir", default_url_path("static/images")), cache_time) + urlmap["/static/scripts"] = Static(conf.get("static_scripts_dir", default_url_path("static/scripts/")), cache_time) + urlmap["/static/style"] = Static(conf.get("static_style_dir", default_url_path("static/style/blue")), cache_time) + urlmap["/static/welcome.html"] = Static(conf.get("static_welcome_html", default_url_path("static/welcome.html")), cache_time) + urlmap["/favicon.ico"] = Static(conf.get("static_favicon_dir", default_url_path("static/favicon.ico")), cache_time) + urlmap["/robots.txt"] = Static(conf.get("static_robots_txt", default_url_path("static/robots.txt")), cache_time) + if 'static_local_dir' in conf: + urlmap["/static_local"] = Static(conf["static_local_dir"], cache_time) return urlmap, cache_time diff --git a/lib/galaxy/webapps/base/controller.py b/lib/galaxy/webapps/base/controller.py index 41c1d66722e..f08c3b81d0b 100644 --- a/lib/galaxy/webapps/base/controller.py +++ b/lib/galaxy/webapps/base/controller.py @@ -546,6 +546,7 @@ class UsesLibraryMixinItems(SharableItemSecurityMixin): # I don't see a reason why hdas copied into libraries should not be visible. # If there is, refactor `ldda.visible = True` to do this only when adding HDCAs. ldda.visible = True + ldda.update_parent_folder_update_times() trans.sa_session.flush() ldda_dict = ldda.to_dict() rval = trans.security.encode_dict_ids(ldda_dict) diff --git a/lib/galaxy/webapps/galaxy/api/folder_contents.py b/lib/galaxy/webapps/galaxy/api/folder_contents.py index a7ecc98481d..da81e0229d8 100644 --- a/lib/galaxy/webapps/galaxy/api/folder_contents.py +++ b/lib/galaxy/webapps/galaxy/api/folder_contents.py @@ -81,13 +81,13 @@ class FolderContentsController(BaseAPIController, UsesLibraryMixin, UsesLibraryM for content_item in self._load_folder_contents(trans, folder, deleted): return_item = {} encoded_id = trans.security.encode_id(content_item.id) - update_time = content_item.update_time.strftime("%Y-%m-%d %I:%M %p") create_time = content_item.create_time.strftime("%Y-%m-%d %I:%M %p") if content_item.api_type == 'folder': encoded_id = 'F' + encoded_id can_modify = is_admin or (trans.user and trans.app.security_agent.can_modify_library_item(current_user_roles, folder)) can_manage = is_admin or (trans.user and trans.app.security_agent.can_manage_library_item(current_user_roles, folder)) + update_time = content_item.update_time.strftime("%Y-%m-%d %I:%M %p") return_item.update(dict(can_modify=can_modify, can_manage=can_manage)) if content_item.description: return_item.update(dict(description=content_item.description)) @@ -107,11 +107,13 @@ class FolderContentsController(BaseAPIController, UsesLibraryMixin, UsesLibraryM can_manage = is_admin or (trans.user and trans.app.security_agent.can_manage_dataset(current_user_roles, content_item.library_dataset_dataset_association.dataset)) raw_size = int(content_item.library_dataset_dataset_association.get_size()) nice_size = util.nice_size(raw_size) + update_time = content_item.library_dataset_dataset_association.update_time.strftime("%Y-%m-%d %I:%M %p") library_dataset_dict = content_item.to_dict() encoded_ldda_id = trans.security.encode_id(content_item.library_dataset_dataset_association.id) return_item.update(dict(file_ext=library_dataset_dict['file_ext'], date_uploaded=library_dataset_dict['date_uploaded'], + update_time=update_time, is_unrestricted=is_unrestricted, is_private=is_private, can_manage=can_manage, diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 0a586fd22be..cee5956577e 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -185,7 +185,7 @@ mapping: tool_config_files (i.e., tool_conf.xml) have been migrated from the Galaxy code distribution to the Tool Shed. This functionality is largely untested in modern Galaxy releases and has serious issues such as #7273 and the possibility - of slowing down Galaxy startup, so the default and recommended value is False. + of slowing down Galaxy startup, so the default and recommended value is false. migrated_tools_config: type: str @@ -283,7 +283,7 @@ mapping: default: false required: false desc: | - Set to True to instruct Galaxy to look for and install missing tool + Set to true to instruct Galaxy to look for and install missing tool dependencies before each job runs. conda_auto_init: @@ -291,7 +291,7 @@ mapping: default: true required: false desc: | - Set to True to instruct Galaxy to install Conda from the web automatically + Set to true to instruct Galaxy to install Conda from the web automatically if it cannot find a local copy and conda_exec is not configured. conda_copy_dependencies: @@ -299,7 +299,7 @@ mapping: default: false required: false desc: | - You must set this to True if conda_prefix and job_working_directory are not on the same + You must set this to true if conda_prefix and job_working_directory are not on the same volume, or some conda dependencies will fail to execute at job runtime. Conda will copy packages content instead of creating hardlinks or symlinks. This will prevent problems with some specific packages (perl, R), at the cost @@ -312,7 +312,7 @@ mapping: desc: | Certain dependency resolvers (namely Conda) take a considerable amount of time to build an isolated job environment in the job_working_directory if the - job working directory is on a network share. Set the following option to True + job working directory is on a network share. Set this option to true to cache the dependencies in a folder. This option is beta and should only be used if you experience long waiting times before a job is actually submitted to your cluster. @@ -336,7 +336,7 @@ mapping: desc: | By default, when using a cached dependency manager, the dependencies are cached when installing new tools and when using tools for the first time. - Set this to False if you prefer dependencies to be cached only when installing new tools. + Set this to false if you prefer dependencies to be cached only when installing new tools. tool_sheds_config_file: type: str @@ -351,7 +351,7 @@ mapping: default: 'false' required: false desc: | - Set to True to enable monitoring of tools and tool directories + Set to true to enable monitoring of tools and tool directories listed in any tool config file specified in tool_config_file option. If changes are found, tools are automatically reloaded. Watchdog ( https://pypi.org/project/watchdog/ ) must be installed and @@ -366,7 +366,7 @@ mapping: default: 'false' required: false desc: | - Set to True to enable monitoring of dynamic job rules. If changes are + Set to true to enable monitoring of dynamic job rules. If changes are found, rules are automatically reloaded. Takes the same values as the 'watch_tools' option. @@ -416,7 +416,7 @@ mapping: dependencies referenced in tools as `requirement`s. The following path is the location of involucro on the Galaxy host. This is ignored if the relevant container resolver isn't enabled, and will install on demand unless - involucro_auto_init is set to False. + involucro_auto_init is set to false. involucro_auto_init: type: bool @@ -505,7 +505,7 @@ mapping: default: 'false' required: false desc: | - Set to True to enable monitoring of the tool_data and shed_tool_data_path + Set to true to enable monitoring of the tool_data and shed_tool_data_path directories. If changes in tool data table files are found, the tool data tables for that data manager are automatically reloaded. Watchdog ( https://pypi.org/project/watchdog/ ) must be installed and @@ -546,7 +546,7 @@ mapping: desc: | Enable sniffing of compressed datatypes. This can be configured/overridden on a per-datatype basis in the datatypes_conf.xml file. - With this option set to False the compressed datatypes will be unpacked + With this option set to false the compressed datatypes will be unpacked before sniffing. @@ -585,7 +585,7 @@ mapping: required: false desc: | To run interactive environment containers in Docker Swarm mode (on an - existing swarm), set this option to True and set `docker_connect_port` in the + existing swarm), set this option to true and set `docker_connect_port` in the IE plugin config (ini) file(s) of any IE plugins you have enabled and ensure that you are not using any `docker run`-specific options in your plugins' `command_inject` options (swarm mode services run using `docker service @@ -823,7 +823,7 @@ mapping: default: false required: false desc: | - User account activation feature global flag. If set to "False", the rest of + User account activation feature global flag. If set to false, the rest of the Account activation configuration is ignored and user activation is disabled (i.e. accounts are active since registration). The activation is also not working in case the SMTP server is not defined. @@ -881,7 +881,7 @@ mapping: which browsers should be available. URLs and builds available at these browsers are defined in the specified files. - If use_remote_user = True, display application servers will be denied access + If use_remote_user is set to true, display application servers will be denied access to Galaxy and so displaying datasets in these sites will fail. display_servers contains a list of hostnames which should be allowed to bypass security to display datasets. Please be aware that there are security @@ -899,8 +899,8 @@ mapping: default: true required: false desc: | - To disable the old-style display applications that are hardcoded into - datatype classes, set enable_old_display_applications = False. + Set this to false to disable the old-style display applications that + are hardcoded into datatype classes. This may be desirable due to using the new-style, XML-defined, display applications that have been defined for many of the datatypes that have the old-style. @@ -1182,7 +1182,7 @@ mapping: https://docs.galaxyproject.org/en/master/admin/production.html Apache can handle file downloads (Galaxy-to-user) via mod_xsendfile. Set - this to True to inform Galaxy that mod_xsendfile is enabled upstream. + this to true to inform Galaxy that mod_xsendfile is enabled upstream. nginx_x_accel_redirect_base: type: str @@ -1273,7 +1273,7 @@ mapping: services based on Galaxy's session cookie. It will attempt to do this by default though you do need to install node+npm and do an npm install from `lib/galaxy/web/proxy/js`. It is generally more robust to configure this - externally, managing it however Galaxy is managed. If True, Galaxy will only + externally, managing it in the same way Galaxy itself is managed. If true, Galaxy will only launch the proxy if it is actually going to be used (e.g. for Jupyter). dynamic_proxy: @@ -1299,7 +1299,7 @@ mapping: required: false desc: | Set the port and IP for the dynamic proxy to bind to, this must match - the external configuration if dynamic_proxy_manage is False. + the external configuration if dynamic_proxy_manage is set to false. dynamic_proxy_bind_ip: type: str @@ -1307,7 +1307,7 @@ mapping: required: false desc: | Set the port and IP for the dynamic proxy to bind to, this must match - the external configuration if dynamic_proxy_manage is False. + the external configuration if dynamic_proxy_manage is set to false. dynamic_proxy_debug: type: bool @@ -1375,7 +1375,7 @@ mapping: default: true required: false desc: | - If True, Galaxy will attempt to configure a simple root logger if a + If true, Galaxy will attempt to configure a simple root logger if a "loggers" section does not appear in this configuration file. log_level: @@ -1389,7 +1389,7 @@ mapping: logging: type: map - allowempty: True + allowempty: true desc: | Controls where and how the server logs messages. If unset, the default is to log all messages to standard output at the level defined by the `log_level` configuration option. Configuration is described in the @@ -1478,7 +1478,7 @@ mapping: By default Galaxy will serve non-HTML tool output that may potentially contain browser executable JavaScript content as plain text. This will for instance cause SVG datasets to not render properly and so may be disabled - by setting the following option to True. + by setting this option to true. allowed_origin_hostnames: type: str @@ -1497,7 +1497,7 @@ mapping: default: false required: false desc: | - Set the following to True to use Jupyter nbconvert to build HTML from Jupyter + Set to true to use Jupyter nbconvert to build HTML from Jupyter notebooks in Galaxy histories. This process may allow users to execute arbitrary code or serve arbitrary HTML. If enabled, Jupyter must be available and on Galaxy's PATH, to do this run @@ -1706,7 +1706,7 @@ mapping: adds an option to the admin library upload tool allowing admins to paste filesystem paths to files and directories in a box, and these paths will be added to a library. For history uploads, this allows pasting in paths as URIs. - (i.e. prefixed with file://). Set to True to enable. Please note the security + (i.e. prefixed with file://). Set to true to enable. Please note the security implication that this will give Galaxy Admins access to anything your Galaxy user has access to. @@ -1902,8 +1902,8 @@ mapping: required: false desc: | If your proxy and/or authentication source does not normalize e-mail - addresses or user names being passed to Galaxy - set the following option - to True to force these to lower case. + addresses or user names being passed to Galaxy - set this option + to true to force these to lower case. single_user: type: str @@ -1939,7 +1939,7 @@ mapping: required: false desc: | Show the site's welcome page (see welcome_url) alongside the login page - (even if require_login is True) + (even if require_login is true). allow_user_creation: type: bool @@ -1961,7 +1961,7 @@ mapping: default: false required: false desc: | - Allow administrators to log in as other users (useful for debugging) + Allow administrators to log in as other users (useful for debugging). show_user_prepopulate_form: type: bool @@ -1985,7 +1985,7 @@ mapping: default: false required: false desc: | - By default, users' data will be public, but setting this to True will cause + By default, users' data will be public, but setting this to true will cause it to be private. Does not affect existing users and data, only ones created after this option is set. Users may still change their default back to public. @@ -1995,30 +1995,30 @@ mapping: default: false required: false desc: | - Expose user list. Setting this to True will expose the user list to + Expose user list. Setting this to true will expose the user list to authenticated users. This makes sharing datasets in smaller galaxy instances much easier as they can type a name/email and have the correct user show up. This makes less sense on large public Galaxy instances where that data shouldn't be exposed. For semi-public Galaxies, it may make sense to expose just the username and not email, or vice versa. - If enable_beta_gdpr is set to True, then this option will be - overridden and set to False. + If enable_beta_gdpr is set to true, then this option will be + overridden and set to false. expose_user_email: type: bool default: false required: false desc: | - Expose user list. Setting this to True will expose the user list to + Expose user list. Setting this to true will expose the user list to authenticated users. This makes sharing datasets in smaller galaxy instances much easier as they can type a name/email and have the correct user show up. This makes less sense on large public Galaxy instances where that data shouldn't be exposed. For semi-public Galaxies, it may make sense to expose just the username and not email, or vice versa. - If enable_beta_gdpr is set to True, then this option will be - overridden and set to False. + If enable_beta_gdpr is set to true, then this option will be + overridden and set to false. fetch_url_whitelist: type: str @@ -2051,13 +2051,6 @@ mapping: Please read the GDPR section under the special topics area of the admin documentation. - enable_beta_ts_api_install: - type: bool - default: true - required: false - desc: | - Enable the new interface for installing tools from Tool Shed - via the API. Admin menu will list both if enabled. enable_beta_containers_interface: type: bool @@ -2235,15 +2228,15 @@ mapping: are selected for "Set at Runtime" inputs from the history such that the same input will not be selected twice, unless there are more inputs than compatible datasets in the history. - When False, the most recently added compatible item in the history will - be used for each "Set at Runtime" input, independent of others in the Workflow + When false, the most recently added compatible item in the history will + be used for each "Set at Runtime" input, independent of others in the workflow. myexperiment_url: type: str default: www.myexperiment.org:80 required: false desc: | - The URL to the myExperiment instance being used (omit scheme but include port) + The URL to the myExperiment instance being used (omit scheme but include port). ftp_upload_dir: type: str @@ -2287,7 +2280,7 @@ mapping: default: true required: false desc: | - This should be set to False to prevent Galaxy from deleting uploaded FTP files + Set to false to prevent Galaxy from deleting uploaded FTP files as it imports them. enable_quotas: @@ -2462,7 +2455,7 @@ mapping: Although it is fairly reliable, setting metadata can occasionally fail. In these instances, you can choose to retry setting it internally or leave it in a failed state (since retrying internally may cause the Galaxy process to be - unresponsive). If this option is set to False, the user will be given the + unresponsive). If this option is set to false, the user will be given the option to retry externally, or set metadata manually (when possible). max_metadata_value_size: @@ -2628,7 +2621,7 @@ mapping: extra database queries must be performed to determine the number of jobs a user has dispatched to a given destination. By default, these queries will happen for every job that is waiting to run, but if cache_user_job_count is - set to True, it will only happen once per iteration of the handler queue. + set to true, it will only happen once per iteration of the handler queue. Although better for performance due to reduced queries, the trade-off is a greater possibility that jobs will be dispatched past the configured limits if running many handlers. diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py index f023ad177bd..800f9f6d5b0 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -851,7 +851,7 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP def check_for_tool_dependencies(self, trans, migration_stage): # Get the 000x_tools.xml file associated with migration_stage. - tools_xml_file_path = os.path.abspath(os.path.join(trans.app.config.root, 'scripts', 'migrate_tools', '%04d_tools.xml' % migration_stage)) + tools_xml_file_path = os.path.abspath(os.path.join(common_util.TOOL_MIGRATION_SCRIPTS_DIR, '%04d_tools.xml' % migration_stage)) tree = util.parse_xml(tools_xml_file_path) root = tree.getroot() tool_shed = root.get('name') @@ -884,6 +884,7 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP message = escape(util.restore_text(kwd.get('message', ''))) status = util.restore_text(kwd.get('status', 'done')) migration_stages_dict = odict() + # FIXME: this isn't valid in an installed context migration_scripts_dir = os.path.abspath(os.path.join(trans.app.config.root, 'lib', 'tool_shed', 'galaxy_install', 'migrate', 'versions')) modules = os.listdir(migration_scripts_dir) modules.sort() diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 8991a39f155..00653846147 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -159,15 +159,6 @@ class AdminToolshed(AdminGalaxy): message=message, status=status) - @web.expose - @web.require_admin - def browse_tool_shed(self, trans, **kwd): - tool_shed_url = kwd.get('tool_shed_url', '') - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(trans.app, tool_shed_url) - params = dict(galaxy_url=web.url_for('/', qualified=True)) - url = util.build_url(tool_shed_url, pathspec=['repository', 'browse_valid_categories'], params=params) - return trans.response.send_redirect(url) - @web.expose @web.require_admin def browse_toolsheds(self, trans, **kwd): @@ -180,14 +171,6 @@ class AdminToolshed(AdminGalaxy): 'app': app, 'bundle': 'extended'}) - @web.expose - @web.require_admin - def browse_tool_sheds(self, trans, **kwd): - message = escape(kwd.get('message', '')) - return trans.fill_template('/webapps/galaxy/admin/tool_sheds.mako', - message=message, - status='error') - @web.expose @web.require_admin def check_for_updates(self, trans, **kwd): @@ -279,30 +262,47 @@ class AdminToolshed(AdminGalaxy): return open(path_to_file, 'rb') return None - @web.expose - @web.require_admin - def find_tools_in_tool_shed(self, trans, **kwd): - tool_shed_url = kwd.get('tool_shed_url', '') - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(trans.app, tool_shed_url) - params = dict(galaxy_url=web.url_for('/', qualified=True)) - url = util.build_url(tool_shed_url, pathspec=['repository', 'find_tools'], params=params) - return trans.response.send_redirect(url) - - @web.expose - @web.require_admin - def find_workflows_in_tool_shed(self, trans, **kwd): - tool_shed_url = kwd.get('tool_shed_url', '') - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(trans.app, tool_shed_url) - params = dict(galaxy_url=web.url_for('/', qualified=True)) - url = util.build_url(tool_shed_url, pathspec=['repository', 'find_workflows'], params=params) - return trans.response.send_redirect(url) - @web.expose @web.require_admin def generate_workflow_image(self, trans, workflow_name, repository_id=None): """Return an svg image representation of a workflow dictionary created when the workflow was exported.""" return workflow_util.generate_workflow_image(trans, workflow_name, repository_metadata_id=None, repository_id=repository_id) + @web.expose + @web.require_admin + def view_tool_metadata(self, trans, repository_id, tool_id, **kwd): + message = escape(kwd.get('message', '')) + status = kwd.get('status', 'done') + repository = repository_util.get_installed_tool_shed_repository(trans.app, repository_id) + repository_metadata = repository.metadata + shed_config_dict = repository.get_shed_config_dict(trans.app) + tool_metadata = {} + tool_lineage = [] + tool = None + if 'tools' in repository_metadata: + for tool_metadata_dict in repository_metadata['tools']: + if tool_metadata_dict['id'] == tool_id: + tool_metadata = tool_metadata_dict + tool_config = tool_metadata['tool_config'] + if shed_config_dict and shed_config_dict.get('tool_path'): + tool_config = os.path.join(shed_config_dict.get('tool_path'), tool_config) + tool = trans.app.toolbox.get_tool(tool_id=tool_metadata['guid'], exact=True) + if not tool: + tool = trans.app.toolbox.load_tool(os.path.abspath(tool_config), guid=tool_metadata['guid']) + if tool: + tool._lineage = trans.app.toolbox._lineage_map.register(tool) + if tool: + tool_lineage = tool.lineage.get_version_ids(reverse=True) + break + return trans.fill_template("/admin/tool_shed_repository/view_tool_metadata.mako", + repository=repository, + repository_metadata=repository_metadata, + tool=tool, + tool_metadata=tool_metadata, + tool_lineage=tool_lineage, + message=message, + status=status) + @web.json @web.require_admin def get_file_contents(self, trans, file_path, repository_id): @@ -1776,41 +1776,6 @@ class AdminToolshed(AdminGalaxy): message=message, status=status)) - @web.expose - @web.require_admin - def view_tool_metadata(self, trans, repository_id, tool_id, **kwd): - message = escape(kwd.get('message', '')) - status = kwd.get('status', 'done') - repository = repository_util.get_installed_tool_shed_repository(trans.app, repository_id) - repository_metadata = repository.metadata - shed_config_dict = repository.get_shed_config_dict(trans.app) - tool_metadata = {} - tool_lineage = [] - tool = None - if 'tools' in repository_metadata: - for tool_metadata_dict in repository_metadata['tools']: - if tool_metadata_dict['id'] == tool_id: - tool_metadata = tool_metadata_dict - tool_config = tool_metadata['tool_config'] - if shed_config_dict and shed_config_dict.get('tool_path'): - tool_config = os.path.join(shed_config_dict.get('tool_path'), tool_config) - tool = trans.app.toolbox.get_tool(tool_id=tool_metadata['guid'], exact=True) - if not tool: - tool = trans.app.toolbox.load_tool(os.path.abspath(tool_config), guid=tool_metadata['guid']) - if tool: - tool._lineage = trans.app.toolbox._lineage_map.register(tool) - if tool: - tool_lineage = tool.lineage.get_version_ids(reverse=True) - break - return trans.fill_template("/admin/tool_shed_repository/view_tool_metadata.mako", - repository=repository, - repository_metadata=repository_metadata, - tool=tool, - tool_metadata=tool_metadata, - tool_lineage=tool_lineage, - message=message, - status=status) - @web.expose @web.require_admin def update_tool_shed_status_for_installed_repository(self, trans, **kwd): diff --git a/lib/galaxy/webapps/tool_shed/config.py b/lib/galaxy/webapps/tool_shed/config.py index 412ee196c49..52e91303fe3 100644 --- a/lib/galaxy/webapps/tool_shed/config.py +++ b/lib/galaxy/webapps/tool_shed/config.py @@ -9,6 +9,7 @@ from datetime import timedelta from six.moves import configparser +from galaxy.config import BaseAppConfiguration from galaxy.util import string_as_bool from galaxy.version import VERSION, VERSION_MAJOR from galaxy.web.formatting import expand_pretty_datetime_format @@ -27,14 +28,17 @@ class ConfigurationError(Exception): pass -class Configuration(object): +class ToolShedAppConfiguration(BaseAppConfiguration): + default_config_file_name = 'tool_shed.yml' def __init__(self, **kwargs): self.config_dict = kwargs self.root = kwargs.get('root_dir', '.') + self._set_config_base(kwargs) + # Resolve paths of other config files - self.__parse_config_file_options(kwargs) + self.parse_config_file_options(kwargs) # Collect the umask and primary gid from the environment self.umask = os.umask(0o77) # get the current umask @@ -43,8 +47,9 @@ class Configuration(object): self.version_major = VERSION_MAJOR self.version = VERSION # Database related configuration - self.database = resolve_path(kwargs.get("database_file", "database/community.sqlite"), self.root) self.database_connection = kwargs.get("database_connection", False) + self.database_connection = kwargs.get("database_connection", + "sqlite:///%s?isolation_level=IMMEDIATE" % resolve_path("community.sqlite", self.data_dir)) self.database_engine_options = get_database_engine_options(kwargs) self.database_create_tables = string_as_bool(kwargs.get("database_create_tables", "True")) # Repository and Tool search API @@ -168,24 +173,14 @@ class Configuration(object): else: return None - def __parse_config_file_options(self, kwargs): - path_list_defaults = dict( - auth_config_file=['config/auth_conf.xml', 'config/auth_conf.xml.sample'], - datatypes_config_file=['config/datatypes_conf.xml', 'datatypes_conf.xml', 'config/datatypes_conf.xml.sample'], - shed_tool_data_table_config=['shed_tool_data_table_conf.xml', 'config/shed_tool_data_table_conf.xml'], + def parse_config_file_options(self, kwargs): + defaults = dict( + auth_config_file=[self._in_config_dir('config/auth_conf.xml')], + datatypes_config_file=[self._in_config_dir('datatypes_conf.xml'), self._in_sample_dir('datatypes_conf.xml.sample')], + shed_tool_data_table_config=[self._in_mutable_config_dir('shed_tool_data_table_conf.xml')], ) - for var, defaults in path_list_defaults.items(): - if kwargs.get(var, None) is not None: - path = kwargs.get(var) - else: - for default in defaults: - if os.path.exists(resolve_path(default, self.root)): - path = default - break - else: - path = defaults[-1] - setattr(self, var, resolve_path(path, self.root)) + self._parse_config_file_options(defaults, dict(), kwargs) # Backwards compatibility for names used in too many places to fix self.datatypes_config = self.datatypes_config_file @@ -229,6 +224,9 @@ class Configuration(object): return user is not None and user.email in admin_users +Configuration = ToolShedAppConfiguration + + def get_database_engine_options(kwargs): """ Allow options for the SQLAlchemy database engine to be passed by using diff --git a/lib/galaxy/workflow/run.py b/lib/galaxy/workflow/run.py index e9120f2529b..3336139eab3 100644 --- a/lib/galaxy/workflow/run.py +++ b/lib/galaxy/workflow/run.py @@ -352,10 +352,16 @@ class WorkflowProgress(object): try: replacement = step_outputs[output_name] except KeyError: - # Must resolve. - template = "Workflow evaluation problem - failed to find output_name %s in step_outputs %s" - message = template % (output_name, step_outputs) - raise Exception(message) + replacement = self.inputs_by_step_id.get(output_step_id) + if connection.output_step.type == 'parameter_input' and output_step_id is not None: + # FIXME: parameter_input step outputs should be properly recorded as step outputs, but for now we can + # short-circuit and just pick the input value + pass + else: + # Must resolve. + template = "Workflow evaluation problem - failed to find output_name %s in step_outputs %s" + message = template % (output_name, step_outputs) + raise Exception(message) if isinstance(replacement, model.HistoryDatasetCollectionAssociation): if not replacement.collection.populated: if not replacement.collection.waiting_for_elements: diff --git a/lib/galaxy/workflow/scheduling_manager.py b/lib/galaxy/workflow/scheduling_manager.py index 41ac0c45a30..d0ea06c0592 100644 --- a/lib/galaxy/workflow/scheduling_manager.py +++ b/lib/galaxy/workflow/scheduling_manager.py @@ -176,7 +176,7 @@ class WorkflowSchedulingManager(ConfiguresHandlers): def __init_schedulers(self): config_file = self.app.config.workflow_schedulers_config_file use_default_scheduler = False - if not config_file: + if not config_file or (not os.path.exists(config_file) and not self.app.config.workflow_schedulers_config_file_set): log.info("No workflow schedulers plugin config file defined, using default scheduler.") use_default_scheduler = True elif not os.path.exists(config_file): diff --git a/lib/log_tempfile.py b/lib/log_tempfile.py deleted file mode 100644 index 3cbb90dd33b..00000000000 --- a/lib/log_tempfile.py +++ /dev/null @@ -1,30 +0,0 @@ -# override tempfile methods for debugging -import logging -import tempfile -import traceback - -log = logging.getLogger(__name__) - - -class TempFile(object): - def __init__(self): - tempfile._NamedTemporaryFile = tempfile.NamedTemporaryFile - tempfile._mkstemp = tempfile.mkstemp - tempfile.NamedTemporaryFile = self.NamedTemporaryFile - tempfile.mkstemp = self.mkstemp - - def NamedTemporaryFile(self, *args, **kwargs): - f = tempfile._NamedTemporaryFile(*args, **kwargs) - try: - log.debug(("Opened tempfile %s with NamedTemporaryFile:\n" % f.name) + "".join(traceback.format_stack())) - except AttributeError: - pass - return f - - def mkstemp(self, *args, **kwargs): - f = tempfile._mkstemp(*args, **kwargs) - try: - log.debug(("Opened tempfile %s with mkstemp:\n" % f[1]) + "".join(traceback.format_stack())) - except TypeError: - pass - return f diff --git a/lib/mimeparse.py b/lib/mimeparse.py deleted file mode 100755 index f38ee04dbae..00000000000 --- a/lib/mimeparse.py +++ /dev/null @@ -1,192 +0,0 @@ -"""MIME-Type Parser - -This module provides basic functions for handling mime-types. It can handle -matching mime-types against a list of media-ranges. See section 14.1 of -the HTTP specification [RFC 2616] for a complete explanation. - - http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.1 - -Contents: - - parse_mime_type(): Parses a mime-type into its component parts. - - parse_media_range(): Media-ranges are mime-types with wild-cards and a 'q' quality parameter. - - quality(): Determines the quality ('q') of a mime-type when compared against a list of media-ranges. - - quality_parsed(): Just like quality() except the second parameter must be pre-parsed. - - best_match(): Choose the mime-type with the highest quality ('q') from a list of candidates. -""" -from functools import reduce - -__version__ = "0.1.2" -__author__ = 'Joe Gregorio' -__email__ = "joe@bitworking.org" -__credits__ = "" - - -def parse_mime_type(mime_type): - """Carves up a mime-type and returns a tuple of the - (type, subtype, params) where 'params' is a dictionary - of all the parameters for the media range. - For example, the media range 'application/xhtml;q=0.5' would - get parsed into: - - ('application', 'xhtml', {'q', '0.5'}) - """ - parts = mime_type.split(";") - params = dict([tuple([s.strip() for s in param.split("=")]) for param in parts[1:]]) - full_type = parts[0].strip() - # Java URLConnection class sends an Accept header that includes a single "*" - # Turn it into a legal wildcard. - if full_type == '*': - full_type = '*/*' - (type, subtype) = full_type.split("/") - return (type.strip(), subtype.strip(), params) - - -def parse_media_range(range): - r""" - Carves up a media range and returns a tuple of the - (type, subtype, params) where 'params' is a dictionary - of all the parameters for the media range. - For example, the media range 'application/*;q=0.5' would - get parsed into: - - .. raw:: text - - ('application', '*', {'q', '0.5'}) - - In addition this function also guarantees that there - is a value for 'q' in the params dictionary, filling it - in with a proper default if necessary. - """ - (type, subtype, params) = parse_mime_type(range) - if 'q' not in params or not params['q'] or \ - not float(params['q']) or float(params['q']) > 1\ - or float(params['q']) < 0: - params['q'] = '1' - return (type, subtype, params) - - -def fitness_and_quality_parsed(mime_type, parsed_ranges): - """Find the best match for a given mime-type against - a list of media_ranges that have already been - parsed by parse_media_range(). Returns a tuple of - the fitness value and the value of the 'q' quality - parameter of the best match, or (-1, 0) if no match - was found. Just as for quality_parsed(), 'parsed_ranges' - must be a list of parsed media ranges. """ - best_fitness = -1 - best_fit_q = 0 - (target_type, target_subtype, target_params) =\ - parse_media_range(mime_type) - for (type, subtype, params) in parsed_ranges: - if (type == target_type or type == '*' or target_type == '*') and \ - (subtype == target_subtype or subtype == '*' or target_subtype == '*'): - param_matches = reduce(lambda x, y: x + y, [1 for (key, value) in - target_params.items() if key != 'q' and - key in params and value == params[key]], 0) - fitness = (type == target_type) and 100 or 0 - fitness += (subtype == target_subtype) and 10 or 0 - fitness += param_matches - if fitness > best_fitness: - best_fitness = fitness - best_fit_q = params['q'] - - return best_fitness, float(best_fit_q) - - -def quality_parsed(mime_type, parsed_ranges): - """Find the best match for a given mime-type against - a list of media_ranges that have already been - parsed by parse_media_range(). Returns the - 'q' quality parameter of the best match, 0 if no - match was found. This function bahaves the same as quality() - except that 'parsed_ranges' must be a list of - parsed media ranges. """ - return fitness_and_quality_parsed(mime_type, parsed_ranges)[1] - - -def quality(mime_type, ranges): - """Returns the quality 'q' of a mime-type when compared - against the media-ranges in ranges. For example: - - >>> quality('text/html','text/*;q=0.3, text/html;q=0.7, text/html;level=1, text/html;level=2;q=0.4, */*;q=0.5') - 0.7 - - """ - parsed_ranges = [parse_media_range(r) for r in ranges.split(",")] - return quality_parsed(mime_type, parsed_ranges) - - -def best_match(supported, header): - """Takes a list of supported mime-types and finds the best - match for all the media-ranges listed in header. The value of - header must be a string that conforms to the format of the - HTTP Accept: header. The value of 'supported' is a list of - mime-types. - - >>> best_match(['application/xbel+xml', 'text/xml'], 'text/*;q=0.5,*/*; q=0.1') - 'text/xml' - """ - parsed_header = [parse_media_range(r) for r in header.split(",")] - weighted_matches = [(fitness_and_quality_parsed(mime_type, parsed_header), mime_type) - for mime_type in supported] - weighted_matches.sort() - return weighted_matches[-1][0][1] and weighted_matches[-1][1] or '' - - -if __name__ == "__main__": - import unittest - - class TestMimeParsing(unittest.TestCase): - - def test_parse_media_range(self): - self.assert_(('application', 'xml', {'q': '1'}) == parse_media_range('application/xml;q=1')) - self.assertEqual(('application', 'xml', {'q': '1'}), parse_media_range('application/xml')) - self.assertEqual(('application', 'xml', {'q': '1'}), parse_media_range('application/xml;q=')) - self.assertEqual(('application', 'xml', {'q': '1'}), parse_media_range('application/xml ; q=')) - self.assertEqual(('application', 'xml', {'q': '1', 'b': 'other'}), parse_media_range('application/xml ; q=1;b=other')) - self.assertEqual(('application', 'xml', {'q': '1', 'b': 'other'}), parse_media_range('application/xml ; q=2;b=other')) - # Java URLConnection class sends an Accept header that includes a single * - self.assertEqual(('*', '*', {'q': '.2'}), parse_media_range(" *; q=.2")) - - def test_rfc_2616_example(self): - accept = "text/*;q=0.3, text/html;q=0.7, text/html;level=1, text/html;level=2;q=0.4, */*;q=0.5" - self.assertEqual(1, quality("text/html;level=1", accept)) - self.assertEqual(0.7, quality("text/html", accept)) - self.assertEqual(0.3, quality("text/plain", accept)) - self.assertEqual(0.5, quality("image/jpeg", accept)) - self.assertEqual(0.4, quality("text/html;level=2", accept)) - self.assertEqual(0.7, quality("text/html;level=3", accept)) - - def test_best_match(self): - mime_types_supported = ['application/xbel+xml', 'application/xml'] - # direct match - self.assertEqual(best_match(mime_types_supported, 'application/xbel+xml'), 'application/xbel+xml') - # direct match with a q parameter - self.assertEqual(best_match(mime_types_supported, 'application/xbel+xml; q=1'), 'application/xbel+xml') - # direct match of our second choice with a q parameter - self.assertEqual(best_match(mime_types_supported, 'application/xml; q=1'), 'application/xml') - # match using a subtype wildcard - self.assertEqual(best_match(mime_types_supported, 'application/*; q=1'), 'application/xml') - # match using a type wildcard - self.assertEqual(best_match(mime_types_supported, '*/*'), 'application/xml') - - mime_types_supported = ['application/xbel+xml', 'text/xml'] - # match using a type versus a lower weighted subtype - self.assertEqual(best_match(mime_types_supported, 'text/*;q=0.5,*/*; q=0.1'), 'text/xml') - # fail to match anything - self.assertEqual(best_match(mime_types_supported, 'text/html,application/atom+xml; q=0.9'), '') - - # common AJAX scenario - mime_types_supported = ['application/json', 'text/html'] - self.assertEqual(best_match(mime_types_supported, 'application/json, text/javascript, */*'), 'application/json') - # verify fitness ordering - self.assertEqual(best_match(mime_types_supported, 'application/json, text/html;q=0.9'), 'application/json') - - def test_support_wildcards(self): - mime_types_supported = ['image/*', 'application/xml'] - # match using a type wildcard - self.assertEqual(best_match(mime_types_supported, 'image/png'), 'image/*') - # match using a wildcard for both requested and supported - self.assertEqual(best_match(mime_types_supported, 'image/*'), 'image/*') - - unittest.main() diff --git a/lib/psyco_full.py b/lib/psyco_full.py deleted file mode 100644 index f18ba8e0758..00000000000 --- a/lib/psyco_full.py +++ /dev/null @@ -1,5 +0,0 @@ -try: - import psyco - psyco.full() -except Exception: - pass diff --git a/lib/tool_shed/galaxy_install/migrate/check.py b/lib/tool_shed/galaxy_install/migrate/check.py index 45ba74fa6b4..b7b8582e18f 100644 --- a/lib/tool_shed/galaxy_install/migrate/check.py +++ b/lib/tool_shed/galaxy_install/migrate/check.py @@ -6,6 +6,7 @@ import sys from migrate.versioning import repository, schema from sqlalchemy import create_engine, MetaData, Table +from galaxy.util import executable from galaxy.util.odict import odict from tool_shed.util import common_util @@ -55,7 +56,8 @@ def verify_tools(app, url, galaxy_config_file=None, engine_options={}): if not app.config.running_functional_tests: if tool_shed_accessible: # Automatically update the value of the migrate_tools.version database table column. - cmd = ['sh', 'manage_db.sh', 'upgrade', 'tools'] + manage_tools = os.path.abspath(os.path.join(os.path.dirname(__file__), 'scripts', 'manage_tools.py')) + cmd = [executable, manage_tools, 'upgrade', 'tools'] if galaxy_config_file: cmd[2:2] = ['-c', galaxy_config_file] proc = subprocess.Popen(args=cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) diff --git a/scripts/migrate_tools/0002_tools.xml b/lib/tool_shed/galaxy_install/migrate/scripts/0002_tools.xml similarity index 100% rename from scripts/migrate_tools/0002_tools.xml rename to lib/tool_shed/galaxy_install/migrate/scripts/0002_tools.xml diff --git a/scripts/migrate_tools/0003_tools.xml b/lib/tool_shed/galaxy_install/migrate/scripts/0003_tools.xml similarity index 100% rename from scripts/migrate_tools/0003_tools.xml rename to lib/tool_shed/galaxy_install/migrate/scripts/0003_tools.xml diff --git a/scripts/migrate_tools/0004_tools.xml b/lib/tool_shed/galaxy_install/migrate/scripts/0004_tools.xml similarity index 100% rename from scripts/migrate_tools/0004_tools.xml rename to lib/tool_shed/galaxy_install/migrate/scripts/0004_tools.xml diff --git a/scripts/migrate_tools/0005_tools.xml b/lib/tool_shed/galaxy_install/migrate/scripts/0005_tools.xml similarity index 100% rename from scripts/migrate_tools/0005_tools.xml rename to lib/tool_shed/galaxy_install/migrate/scripts/0005_tools.xml diff --git a/scripts/migrate_tools/0006_tools.xml b/lib/tool_shed/galaxy_install/migrate/scripts/0006_tools.xml similarity index 100% rename from scripts/migrate_tools/0006_tools.xml rename to lib/tool_shed/galaxy_install/migrate/scripts/0006_tools.xml diff --git a/scripts/migrate_tools/0007_tools.xml b/lib/tool_shed/galaxy_install/migrate/scripts/0007_tools.xml similarity index 100% rename from scripts/migrate_tools/0007_tools.xml rename to lib/tool_shed/galaxy_install/migrate/scripts/0007_tools.xml diff --git a/scripts/migrate_tools/0008_tools.xml b/lib/tool_shed/galaxy_install/migrate/scripts/0008_tools.xml similarity index 100% rename from scripts/migrate_tools/0008_tools.xml rename to lib/tool_shed/galaxy_install/migrate/scripts/0008_tools.xml diff --git a/scripts/migrate_tools/0009_tools.xml b/lib/tool_shed/galaxy_install/migrate/scripts/0009_tools.xml similarity index 100% rename from scripts/migrate_tools/0009_tools.xml rename to lib/tool_shed/galaxy_install/migrate/scripts/0009_tools.xml diff --git a/scripts/migrate_tools/0010_tools.xml b/lib/tool_shed/galaxy_install/migrate/scripts/0010_tools.xml similarity index 100% rename from scripts/migrate_tools/0010_tools.xml rename to lib/tool_shed/galaxy_install/migrate/scripts/0010_tools.xml diff --git a/scripts/migrate_tools/0011_tools.xml b/lib/tool_shed/galaxy_install/migrate/scripts/0011_tools.xml similarity index 100% rename from scripts/migrate_tools/0011_tools.xml rename to lib/tool_shed/galaxy_install/migrate/scripts/0011_tools.xml diff --git a/scripts/migrate_tools/0012_tools.xml b/lib/tool_shed/galaxy_install/migrate/scripts/0012_tools.xml similarity index 100% rename from scripts/migrate_tools/0012_tools.xml rename to lib/tool_shed/galaxy_install/migrate/scripts/0012_tools.xml diff --git a/scripts/migrate_tools/migrate_tools.py b/lib/tool_shed/galaxy_install/migrate/scripts/migrate_tools.py similarity index 59% rename from scripts/migrate_tools/migrate_tools.py rename to lib/tool_shed/galaxy_install/migrate/scripts/migrate_tools.py index ea2ae32aab3..749e5a8d2a1 100644 --- a/scripts/migrate_tools/migrate_tools.py +++ b/lib/tool_shed/galaxy_install/migrate/scripts/migrate_tools.py @@ -12,20 +12,26 @@ from __future__ import print_function import os import sys -sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib'))) +sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, os.pardir))) from tool_shed.galaxy_install.migrate.common import MigrateToolsApplication -app = MigrateToolsApplication(sys.argv[1]) -non_shed_tool_confs = app.tool_migration_manager.proprietary_tool_confs -if len(non_shed_tool_confs) == 1: - plural = '' - file_names = non_shed_tool_confs[0] -else: - plural = 's' - file_names = ', '.join(non_shed_tool_confs) -msg = "\nThe installation process is finished. All tools associated with this migration that were defined in your file%s named\n" % plural -msg += "%s, have been removed. You may now start your Galaxy server.\n" % file_names -print(msg) -app.shutdown() -sys.exit(0) + +def main(): + app = MigrateToolsApplication(sys.argv[1]) + non_shed_tool_confs = app.tool_migration_manager.proprietary_tool_confs + if len(non_shed_tool_confs) == 1: + plural = '' + file_names = non_shed_tool_confs[0] + else: + plural = 's' + file_names = ', '.join(non_shed_tool_confs) + msg = "\nThe installation process is finished. All tools associated with this migration that were defined in your file%s named\n" % plural + msg += "%s, have been removed. You may now start your Galaxy server.\n" % file_names + print(msg) + app.shutdown() + sys.exit(0) + + +if __name__ == '__main__': + main() diff --git a/lib/tool_shed/galaxy_install/tool_migration_manager.py b/lib/tool_shed/galaxy_install/tool_migration_manager.py index 945b433f111..e08f8227fb2 100644 --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -2,6 +2,7 @@ Manage automatic installation of tools configured in the xxx.xml files in ~/scripts/migrate_tools (e.g., 0002_tools.xml). All of the tools were at some point included in the Galaxy distribution, but are now hosted in the main Galaxy tool shed. """ +import errno import logging import os import shutil @@ -31,6 +32,12 @@ from tool_shed.util import ( log = logging.getLogger(__name__) +MIGRATED_TOOLS_CONF_XML = """ + + +""" + + class ToolMigrationManager(object): def __init__(self, app, latest_migration_script_number, tool_shed_install_config, migrated_tools_config, @@ -59,7 +66,15 @@ class ToolMigrationManager(object): self.proprietary_tool_panel_elems = self.get_proprietary_tool_panel_elems(latest_migration_script_number) # Set the location where the repositories will be installed by retrieving the tool_path # setting from migrated_tools_config. - tree, error_message = xml_util.parse_xml(migrated_tools_config) + try: + tree, error_message = xml_util.parse_xml(migrated_tools_config) + except (IOError, OSError) as exc: + if exc.errno == errno.ENOENT: + with open(migrated_tools_config, 'w') as fh: + fh.write(MIGRATED_TOOLS_CONF_XML.format(shed_tools_dir=self.app.config.shed_tools_dir)) + tree, error_message = xml_util.parse_xml(migrated_tools_config) + else: + raise if tree is None: log.error(error_message) else: diff --git a/lib/tool_shed/galaxy_install/tools/data_manager.py b/lib/tool_shed/galaxy_install/tools/data_manager.py index 967def810ee..279e90b72d6 100644 --- a/lib/tool_shed/galaxy_install/tools/data_manager.py +++ b/lib/tool_shed/galaxy_install/tools/data_manager.py @@ -1,3 +1,4 @@ +import errno import logging import os import time @@ -10,6 +11,11 @@ from tool_shed.util import xml_util log = logging.getLogger(__name__) +SHED_DATA_MANAGER_CONF_XML = """ + + +""" + class DataManagerHandler(object): @@ -52,7 +58,15 @@ class DataManagerHandler(object): for tool_tup in repository_tools_tups: repository_tools_by_guid[tool_tup[1]] = dict(tool_config_filename=tool_tup[0], tool=tool_tup[2]) # Load existing data managers. - tree, error_message = xml_util.parse_xml(shed_data_manager_conf_filename) + try: + tree, error_message = xml_util.parse_xml(shed_data_manager_conf_filename) + except (OSError, IOError) as exc: + if exc.errno == errno.ENOENT: + with open(shed_data_manager_conf_filename, 'w') as fh: + fh.write(SHED_DATA_MANAGER_CONF_XML) + tree, error_message = xml_util.parse_xml(shed_data_manager_conf_filename) + else: + raise if tree is None: return rval config_elems = [elem for elem in tree.getroot()] diff --git a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py index 9c1c76a004a..c5a167bb57a 100644 --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -1,3 +1,4 @@ +import errno import logging from xml.etree import ElementTree as XmlET @@ -31,7 +32,15 @@ class ToolPanelManager(object): shed_tool_conf = shed_tool_conf_dict['config_filename'] tool_path = shed_tool_conf_dict['tool_path'] config_elems = [] - tree, error_message = xml_util.parse_xml(shed_tool_conf) + try: + tree, error_message = xml_util.parse_xml(shed_tool_conf) + except (OSError, IOError) as exc: + if (exc.errno == errno.ENOENT and shed_tool_conf_dict.get('create', None) is not None): + with open(shed_tool_conf, 'w') as fh: + fh.write(shed_tool_conf_dict['create']) + tree, error_message = xml_util.parse_xml(shed_tool_conf) + else: + raise if tree: root = tree.getroot() for elem in root: diff --git a/lib/tool_shed/tool_shed_registry.py b/lib/tool_shed/tool_shed_registry.py index a8ff5626e71..80528095df9 100644 --- a/lib/tool_shed/tool_shed_registry.py +++ b/lib/tool_shed/tool_shed_registry.py @@ -1,4 +1,5 @@ import logging +import xml.etree.ElementTree from six.moves.urllib import request as urlrequest @@ -7,6 +8,12 @@ from tool_shed.util import common_util, xml_util log = logging.getLogger(__name__) +DEFAULT_TOOL_SHEDS_CONF_XML = """ + + + +""" + class Registry(object): @@ -18,25 +25,28 @@ class Registry(object): tree, error_message = xml_util.parse_xml(config) if tree is None: log.warning("Unable to load references to tool sheds defined in file %s" % str(config)) - else: - root = tree.getroot() - log.debug('Loading references to tool sheds from %s' % config) - for elem in root.findall('tool_shed'): - try: - name = elem.get('name', None) - url = elem.get('url', None) - username = elem.get('user', None) - password = elem.get('pass', None) - if name and url: - self.tool_sheds[name] = url - self.tool_sheds_auth[name] = None - log.debug('Loaded reference to tool shed: %s' % name) - if name and url and username and password: - pass_mgr = urlrequest.HTTPPasswordMgrWithDefaultRealm() - pass_mgr.add_password(None, url, username, password) - self.tool_sheds_auth[name] = pass_mgr - except Exception as e: - log.warning('Error loading reference to tool shed "%s", problem: %s' % (name, str(e))) + return + root = tree.getroot() + else: + root = xml.etree.ElementTree.fromstring(DEFAULT_TOOL_SHEDS_CONF_XML) + config = "internal default config" + log.debug('Loading references to tool sheds from %s' % config) + for elem in root.findall('tool_shed'): + try: + name = elem.get('name', None) + url = elem.get('url', None) + username = elem.get('user', None) + password = elem.get('pass', None) + if name and url: + self.tool_sheds[name] = url + self.tool_sheds_auth[name] = None + log.debug('Loaded reference to tool shed: %s' % name) + if name and url and username and password: + pass_mgr = urlrequest.HTTPPasswordMgrWithDefaultRealm() + pass_mgr.add_password(None, url, username, password) + self.tool_sheds_auth[name] = pass_mgr + except Exception as e: + log.warning('Error loading reference to tool shed "%s", problem: %s' % (name, str(e))) def password_manager_for_url(self, url): """ diff --git a/lib/tool_shed/util/common_util.py b/lib/tool_shed/util/common_util.py deleted file mode 100644 index 8f5686197a2..00000000000 --- a/lib/tool_shed/util/common_util.py +++ /dev/null @@ -1,362 +0,0 @@ -import json -import logging -import os - -from six.moves.urllib.parse import urljoin - -from galaxy import util -from galaxy.util.odict import odict -from galaxy.web import url_for -from tool_shed.util import encoding_util, xml_util - -log = logging.getLogger(__name__) - -REPOSITORY_OWNER = 'devteam' - - -def accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies): - if tool_shed_accessible: - if tool_dependencies: - for tool_dependency in tool_dependencies: - if tool_dependency not in all_tool_dependencies: - all_tool_dependencies.append(tool_dependency) - return all_tool_dependencies - - -def check_for_missing_tools(app, tool_panel_configs, latest_tool_migration_script_number): - # Get the 000x_tools.xml file associated with the current migrate_tools version number. - tools_xml_file_path = os.path.abspath(os.path.join('scripts', 'migrate_tools', '%04d_tools.xml' % latest_tool_migration_script_number)) - # Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config. - migrated_tool_configs_dict = odict() - tree, error_message = xml_util.parse_xml(tools_xml_file_path) - if tree is None: - return False, odict() - root = tree.getroot() - tool_shed = root.get('name') - tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, tool_shed) - # The default behavior is that the tool shed is down. - tool_shed_accessible = False - missing_tool_configs_dict = odict() - if tool_shed_url: - for elem in root: - if elem.tag == 'repository': - repository_dependencies = [] - all_tool_dependencies = [] - repository_name = elem.get('name') - changeset_revision = elem.get('changeset_revision') - tool_shed_accessible, repository_dependencies_dict = get_repository_dependencies(app, - tool_shed_url, - repository_name, - REPOSITORY_OWNER, - changeset_revision) - if tool_shed_accessible: - # Accumulate all tool dependencies defined for repository dependencies for display to the user. - for rd_key, rd_tups in repository_dependencies_dict.items(): - if rd_key in ['root_key', 'description']: - continue - for rd_tup in rd_tups: - tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - parse_repository_dependency_tuple(rd_tup) - tool_shed_accessible, tool_dependencies = get_tool_dependencies(app, - tool_shed_url, - name, - owner, - changeset_revision) - all_tool_dependencies = accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies) - tool_shed_accessible, tool_dependencies = get_tool_dependencies(app, - tool_shed_url, - repository_name, - REPOSITORY_OWNER, - changeset_revision) - all_tool_dependencies = accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies) - for tool_elem in elem.findall('tool'): - tool_config_file_name = tool_elem.get('file') - if tool_config_file_name: - # We currently do nothing with repository dependencies except install them (we do not display repositories that will be - # installed to the user). However, we'll store them in the following dictionary in case we choose to display them in the - # future. - dependencies_dict = dict(tool_dependencies=all_tool_dependencies, - repository_dependencies=repository_dependencies) - migrated_tool_configs_dict[tool_config_file_name] = dependencies_dict - else: - break - if tool_shed_accessible: - # Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names. - for tool_panel_config in tool_panel_configs: - tree, error_message = xml_util.parse_xml(tool_panel_config) - if tree: - root = tree.getroot() - for elem in root: - if elem.tag == 'tool': - missing_tool_configs_dict = check_tool_tag_set(elem, migrated_tool_configs_dict, missing_tool_configs_dict) - elif elem.tag == 'section': - for section_elem in elem: - if section_elem.tag == 'tool': - missing_tool_configs_dict = check_tool_tag_set(section_elem, migrated_tool_configs_dict, missing_tool_configs_dict) - else: - exception_msg = '\n\nThe entry for the main Galaxy tool shed at %s is missing from the %s file. ' % (tool_shed, app.config.tool_sheds_config) - exception_msg += 'The entry for this tool shed must always be available in this file, so re-add it before attempting to start your Galaxy server.\n' - raise Exception(exception_msg) - return tool_shed_accessible, missing_tool_configs_dict - - -def check_tool_tag_set(elem, migrated_tool_configs_dict, missing_tool_configs_dict): - file_path = elem.get('file', None) - if file_path: - name = os.path.basename(file_path) - for migrated_tool_config in migrated_tool_configs_dict.keys(): - if migrated_tool_config in [file_path, name]: - missing_tool_configs_dict[name] = migrated_tool_configs_dict[migrated_tool_config] - return missing_tool_configs_dict - - -def generate_clone_url_for_installed_repository(app, repository): - """Generate the URL for cloning a repository that has been installed into a Galaxy instance.""" - tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, str(repository.tool_shed)) - return util.build_url(tool_shed_url, pathspec=['repos', str(repository.owner), str(repository.name)]) - - -def generate_clone_url_for_repository_in_tool_shed(user, repository): - """Generate the URL for cloning a repository that is in the tool shed.""" - base_url = url_for('/', qualified=True).rstrip('/') - if user: - protocol, base = base_url.split('://') - username = '%s@' % user.username - return '%s://%s%s/repos/%s/%s' % (protocol, username, base, repository.user.username, repository.name) - else: - return '%s/repos/%s/%s' % (base_url, repository.user.username, repository.name) - - -def generate_clone_url_from_repo_info_tup(app, repo_info_tup): - """Generate the URL for cloning a repository given a tuple of toolshed, name, owner, changeset_revision.""" - # Example tuple: ['http://localhost:9009', 'blast_datatypes', 'test', '461a4216e8ab', False] - toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - parse_repository_dependency_tuple(repo_info_tup) - tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, toolshed) - # Don't include the changeset_revision in clone urls. - return util.build_url(tool_shed_url, pathspec=['repos', owner, name]) - - -def get_non_shed_tool_panel_configs(app): - """Get the non-shed related tool panel configs - there can be more than one, and the default is tool_conf.xml.""" - config_filenames = [] - for config_filename in app.config.tool_configs: - # Any config file that includes a tool_path attribute in the root tag set like the following is shed-related. - # - tree, error_message = xml_util.parse_xml(config_filename) - if tree is None: - continue - root = tree.getroot() - tool_path = root.get('tool_path', None) - if tool_path is None: - config_filenames.append(config_filename) - return config_filenames - - -def get_repository_dependencies(app, tool_shed_url, repository_name, repository_owner, changeset_revision): - repository_dependencies_dict = {} - tool_shed_accessible = True - params = dict(name=repository_name, owner=repository_owner, changeset_revision=changeset_revision) - pathspec = ['repository', 'get_repository_dependencies'] - try: - raw_text = util.url_get(tool_shed_url, password_mgr=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params) - tool_shed_accessible = True - except Exception as e: - tool_shed_accessible = False - log.warning("The URL\n%s\nraised the exception:\n%s\n", util.build_url(tool_shed_url, pathspec=pathspec, params=params), e) - if tool_shed_accessible: - if len(raw_text) > 2: - encoded_text = json.loads(util.unicodify(raw_text)) - repository_dependencies_dict = encoding_util.tool_shed_decode(encoded_text) - return tool_shed_accessible, repository_dependencies_dict - - -def get_protocol_from_tool_shed_url(tool_shed_url): - """Return the protocol from the received tool_shed_url if it exists.""" - try: - if tool_shed_url.find('://') > 0: - return tool_shed_url.split('://')[0].lower() - except Exception: - # We receive a lot of calls here where the tool_shed_url is None. The container_util uses - # that value when creating a header row. If the tool_shed_url is not None, we have a problem. - if tool_shed_url is not None: - log.exception("Handled exception getting the protocol from Tool Shed URL %s", str(tool_shed_url)) - # Default to HTTP protocol. - return 'http' - - -def get_tool_dependencies(app, tool_shed_url, repository_name, repository_owner, changeset_revision): - tool_dependencies = [] - tool_shed_accessible = True - params = dict(name=repository_name, owner=repository_owner, changeset_revision=changeset_revision) - pathspec = ['repository', 'get_tool_dependencies'] - try: - text = util.url_get(tool_shed_url, password_mgr=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params) - tool_shed_accessible = True - except Exception as e: - tool_shed_accessible = False - log.warning("The URL\n%s\nraised the exception:\n%s\n", util.build_url(tool_shed_url, pathspec=pathspec, params=params), e) - if tool_shed_accessible: - if text: - tool_dependencies_dict = encoding_util.tool_shed_decode(text) - for requirements_dict in tool_dependencies_dict.values(): - tool_dependency_name = requirements_dict['name'] - tool_dependency_version = requirements_dict['version'] - tool_dependency_type = requirements_dict['type'] - tool_dependencies.append((tool_dependency_name, tool_dependency_version, tool_dependency_type)) - return tool_shed_accessible, tool_dependencies - - -def get_tool_shed_repository_ids(as_string=False, **kwd): - tsrid = kwd.get('tool_shed_repository_id', None) - tsridslist = util.listify(kwd.get('tool_shed_repository_ids', None)) - if not tsridslist: - tsridslist = util.listify(kwd.get('id', None)) - if tsridslist is not None: - if tsrid is not None and tsrid not in tsridslist: - tsridslist.append(tsrid) - if as_string: - return ','.join(tsridslist) - return tsridslist - else: - tsridslist = util.listify(kwd.get('ordered_tsr_ids', None)) - if tsridslist is not None: - if as_string: - return ','.join(tsridslist) - return tsridslist - if as_string: - return '' - return [] - - -def get_tool_shed_url_from_tool_shed_registry(app, tool_shed): - """ - The value of tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is - something like: http://toolshed.g2.bx.psu.edu/ - """ - cleaned_tool_shed = remove_protocol_from_tool_shed_url(tool_shed) - for shed_url in app.tool_shed_registry.tool_sheds.values(): - if shed_url.find(cleaned_tool_shed) >= 0: - if shed_url.endswith('/'): - shed_url = shed_url.rstrip('/') - return shed_url - # The tool shed from which the repository was originally installed must no longer be configured in tool_sheds_conf.xml. - return None - - -def get_tool_shed_repository_url(app, tool_shed, owner, name): - tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, tool_shed) - if tool_shed_url: - # Append a slash to the tool shed URL, because urlparse.urljoin will eliminate - # the last part of a URL if it does not end with a forward slash. - tool_shed_url = '%s/' % tool_shed_url - return urljoin(tool_shed_url, 'view/%s/%s' % (owner, name)) - return tool_shed_url - - -def get_user_by_username(app, username): - """Get a user from the database by username.""" - sa_session = app.model.context.current - try: - user = sa_session.query(app.model.User) \ - .filter(app.model.User.table.c.username == username) \ - .one() - return user - except Exception: - return None - - -def handle_galaxy_url(trans, **kwd): - galaxy_url = kwd.get('galaxy_url', None) - if galaxy_url: - trans.set_cookie(galaxy_url, name='toolshedgalaxyurl') - else: - galaxy_url = trans.get_cookie(name='toolshedgalaxyurl') - return galaxy_url - - -def handle_tool_shed_url_protocol(app, shed_url): - """Handle secure and insecure HTTP protocol since they may change over time.""" - try: - if app.name == 'galaxy': - url = remove_protocol_from_tool_shed_url(shed_url) - tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, url) - else: - tool_shed_url = str(url_for('/', qualified=True)).rstrip('/') - return tool_shed_url - except Exception: - # We receive a lot of calls here where the tool_shed_url is None. The container_util uses - # that value when creating a header row. If the tool_shed_url is not None, we have a problem. - if shed_url is not None: - log.exception("Handled exception removing protocol from URL %s", str(shed_url)) - return shed_url - - -def parse_repository_dependency_tuple(repository_dependency_tuple, contains_error=False): - # Default both prior_installation_required and only_if_compiling_contained_td to False in cases where metadata should be reset on the - # repository containing the repository_dependency definition. - prior_installation_required = 'False' - only_if_compiling_contained_td = 'False' - if contains_error: - if len(repository_dependency_tuple) == 5: - tool_shed, name, owner, changeset_revision, error = repository_dependency_tuple - elif len(repository_dependency_tuple) == 6: - tool_shed, name, owner, changeset_revision, prior_installation_required, error = repository_dependency_tuple - elif len(repository_dependency_tuple) == 7: - tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, error = \ - repository_dependency_tuple - return tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, error - else: - if len(repository_dependency_tuple) == 4: - tool_shed, name, owner, changeset_revision = repository_dependency_tuple - elif len(repository_dependency_tuple) == 5: - tool_shed, name, owner, changeset_revision, prior_installation_required = repository_dependency_tuple - elif len(repository_dependency_tuple) == 6: - tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = repository_dependency_tuple - return tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td - - -def remove_port_from_tool_shed_url(tool_shed_url): - """Return a partial Tool Shed URL, eliminating the port if it exists.""" - try: - if tool_shed_url.find(':') > 0: - # Eliminate the port, if any, since it will result in an invalid directory name. - new_tool_shed_url = tool_shed_url.split(':')[0] - else: - new_tool_shed_url = tool_shed_url - return new_tool_shed_url.rstrip('/') - except Exception: - # We receive a lot of calls here where the tool_shed_url is None. The container_util uses - # that value when creating a header row. If the tool_shed_url is not None, we have a problem. - if tool_shed_url is not None: - log.exception("Handled exception removing the port from Tool Shed URL %s", str(tool_shed_url)) - return tool_shed_url - - -def remove_protocol_and_port_from_tool_shed_url(tool_shed_url): - """Return a partial Tool Shed URL, eliminating the protocol and/or port if either exists.""" - tool_shed = remove_protocol_from_tool_shed_url(tool_shed_url) - tool_shed = remove_port_from_tool_shed_url(tool_shed) - return tool_shed - - -def remove_protocol_and_user_from_clone_url(repository_clone_url): - """Return a URL that can be used to clone a repository, eliminating the protocol and user if either exists.""" - if repository_clone_url.find('@') > 0: - # We have an url that includes an authenticated user, something like: - # http://test@bx.psu.edu:9009/repos/some_username/column - items = repository_clone_url.split('@') - tmp_url = items[1] - elif repository_clone_url.find('//') > 0: - # We have an url that includes only a protocol, something like: - # http://bx.psu.edu:9009/repos/some_username/column - items = repository_clone_url.split('//') - tmp_url = items[1] - else: - tmp_url = repository_clone_url - return tmp_url.rstrip('/') - - -def remove_protocol_from_tool_shed_url(tool_shed_url): - """Return a partial Tool Shed URL, eliminating the protocol if it exists.""" - return util.remove_protocol_from_url(tool_shed_url) diff --git a/lib/tool_shed/util/common_util.py b/lib/tool_shed/util/common_util.py new file mode 120000 index 00000000000..71148b57d1c --- /dev/null +++ b/lib/tool_shed/util/common_util.py @@ -0,0 +1 @@ +../../galaxy/util/tool_shed/common_util.py \ No newline at end of file diff --git a/lib/tool_shed/util/encoding_util.py b/lib/tool_shed/util/encoding_util.py deleted file mode 100644 index f5bb2ff6441..00000000000 --- a/lib/tool_shed/util/encoding_util.py +++ /dev/null @@ -1,43 +0,0 @@ -import binascii -import json -import logging - -from galaxy.util import ( - smart_str, - unicodify -) -from galaxy.util.hash_util import hmac_new - -log = logging.getLogger(__name__) - -encoding_sep = '__esep__' -encoding_sep2 = '__esepii__' - - -def tool_shed_decode(value): - # Extract and verify hash - value = unicodify(value) - a, b = value.split(":") - value = binascii.unhexlify(b) - test = hmac_new(b'ToolShedAndGalaxyMustHaveThisSameKey', value) - assert a == test - # Restore from string - values = None - value = unicodify(value) - try: - values = json.loads(value) - except Exception: - pass - if values is None: - values = value - return values - - -def tool_shed_encode(val): - if isinstance(val, dict) or isinstance(val, list): - value = json.dumps(val) - else: - value = val - a = hmac_new(b'ToolShedAndGalaxyMustHaveThisSameKey', smart_str(value)) - b = unicodify(binascii.hexlify(smart_str(value))) - return "%s:%s" % (a, b) diff --git a/lib/tool_shed/util/encoding_util.py b/lib/tool_shed/util/encoding_util.py new file mode 120000 index 00000000000..5bb58723d83 --- /dev/null +++ b/lib/tool_shed/util/encoding_util.py @@ -0,0 +1 @@ +../../galaxy/util/tool_shed/encoding_util.py \ No newline at end of file diff --git a/lib/tool_shed/util/xml_util.py b/lib/tool_shed/util/xml_util.py deleted file mode 100644 index 5e7e9db05e4..00000000000 --- a/lib/tool_shed/util/xml_util.py +++ /dev/null @@ -1,92 +0,0 @@ -import io -import logging -import os -import tempfile -from xml.etree import ElementTree as XmlET - -from galaxy.util import ( - xml_to_string -) - -log = logging.getLogger(__name__) - - -class Py27CommentedTreeBuilder(XmlET.TreeBuilder): - - def doctype(*args): - # handle deprecation warning for XMLParsing a file with DOCTYPE - pass - - def comment(self, data): - self.start(XmlET.Comment, {}) - self.data(data) - self.end(XmlET.Comment) - - -def create_and_write_tmp_file(elem): - tmp_str = xml_to_string(elem, pretty=True) - with tempfile.NamedTemporaryFile(prefix="tmp-toolshed-cawrf", delete=False) as fh: - tmp_filename = fh.name - with io.open(tmp_filename, mode='w', encoding='utf-8') as fh: - fh.write(tmp_str) - return tmp_filename - - -def create_element(tag, attributes=None, sub_elements=None): - """ - Create a new element whose tag is the value of the received tag, and whose attributes are all - key / value pairs in the received attributes and sub_elements. - """ - if tag: - elem = XmlET.Element(tag) - if attributes: - # The received attributes is an odict to preserve ordering. - for k, v in attributes.items(): - elem.set(k, v) - if sub_elements: - # The received attributes is an odict. These handle information that tends to be - # long text including paragraphs (e.g., description and long_description. - for k, v in sub_elements.items(): - # Don't include fields that are blank. - if v: - if k == 'packages': - # The received sub_elements is an odict whose key is 'packages' and whose - # value is a list of ( name, version ) tuples. - for v_tuple in v: - sub_elem = XmlET.SubElement(elem, 'package') - sub_elem_name, sub_elem_version = v_tuple - sub_elem.set('name', sub_elem_name) - sub_elem.set('version', sub_elem_version) - elif isinstance(v, list): - sub_elem = XmlET.SubElement(elem, k) - # If v is a list, then it must be a list of tuples where the first - # item is the tag and the second item is the text value. - for v_tuple in v: - if len(v_tuple) == 2: - v_tag = v_tuple[0] - v_text = v_tuple[1] - # Don't include fields that are blank. - if v_text: - v_elem = XmlET.SubElement(sub_elem, v_tag) - v_elem.text = v_text - else: - sub_elem = XmlET.SubElement(elem, k) - sub_elem.text = v - return elem - return None - - -def parse_xml(file_name): - """Returns a parsed xml tree with comments intact.""" - error_message = '' - if not os.path.exists(file_name): - return None, "File does not exist %s" % str(file_name) - - with open(file_name, 'r') as fobj: - try: - tree = XmlET.parse(fobj, parser=XmlET.XMLParser(target=Py27CommentedTreeBuilder())) - except Exception as e: - error_message = "Exception attempting to parse %s: %s" % (str(file_name), str(e)) - log.exception(error_message) - return None, error_message - return tree, error_message diff --git a/lib/tool_shed/util/xml_util.py b/lib/tool_shed/util/xml_util.py new file mode 120000 index 00000000000..cb738288301 --- /dev/null +++ b/lib/tool_shed/util/xml_util.py @@ -0,0 +1 @@ +../../galaxy/util/tool_shed/xml_util.py \ No newline at end of file diff --git a/packages/app/galaxy/app.py b/packages/app/galaxy/app.py new file mode 120000 index 00000000000..61ec0a3dd4d --- /dev/null +++ b/packages/app/galaxy/app.py @@ -0,0 +1 @@ +../../../lib/galaxy/app.py \ No newline at end of file diff --git a/packages/app/galaxy/config.py b/packages/app/galaxy/config.py new file mode 120000 index 00000000000..7e61acb3d31 --- /dev/null +++ b/packages/app/galaxy/config.py @@ -0,0 +1 @@ +../../../lib/galaxy/config.py \ No newline at end of file diff --git a/packages/app/galaxy/config_watchers.py b/packages/app/galaxy/config_watchers.py new file mode 120000 index 00000000000..ffd3e6bd109 --- /dev/null +++ b/packages/app/galaxy/config_watchers.py @@ -0,0 +1 @@ +../../../lib/galaxy/config_watchers.py \ No newline at end of file diff --git a/packages/app/galaxy/main.py b/packages/app/galaxy/main.py new file mode 120000 index 00000000000..6b458fd3ff8 --- /dev/null +++ b/packages/app/galaxy/main.py @@ -0,0 +1 @@ +../../../lib/galaxy/main.py \ No newline at end of file diff --git a/packages/app/galaxy/queue_worker.py b/packages/app/galaxy/queue_worker.py new file mode 120000 index 00000000000..7fcb68fbea9 --- /dev/null +++ b/packages/app/galaxy/queue_worker.py @@ -0,0 +1 @@ +../../../lib/galaxy/queue_worker.py \ No newline at end of file diff --git a/packages/app/galaxy/queues.py b/packages/app/galaxy/queues.py new file mode 120000 index 00000000000..a5afab45290 --- /dev/null +++ b/packages/app/galaxy/queues.py @@ -0,0 +1 @@ +../../../lib/galaxy/queues.py \ No newline at end of file diff --git a/packages/app/galaxy/version.py b/packages/app/galaxy/version.py new file mode 120000 index 00000000000..42d972d9994 --- /dev/null +++ b/packages/app/galaxy/version.py @@ -0,0 +1 @@ +../../../lib/galaxy/version.py \ No newline at end of file diff --git a/packages/auth/tests/test_auth.py b/packages/auth/tests/test_auth.py new file mode 120000 index 00000000000..58ff6f86bf1 --- /dev/null +++ b/packages/auth/tests/test_auth.py @@ -0,0 +1 @@ +../../../test/unit/test_auth.py \ No newline at end of file diff --git a/packages/containers/tests/test_containers.py b/packages/containers/tests/test_containers.py new file mode 120000 index 00000000000..2fb68299fa1 --- /dev/null +++ b/packages/containers/tests/test_containers.py @@ -0,0 +1 @@ +../../../test/unit/test_containers.py \ No newline at end of file diff --git a/packages/data/requirements.txt b/packages/data/requirements.txt index 8cebfd65d71..40d2a9baed0 100644 --- a/packages/data/requirements.txt +++ b/packages/data/requirements.txt @@ -1,13 +1,16 @@ galaxy-objectstore -galaxy-util +galaxy-sequence-utils==1.1.3 +galaxy-util[template] bdbag bx-python h5py isa-rwval +parsley numpy<=1.16 pycryptodome pysam social_auth_core SQLAlchemy==1.2 sqlalchemy-migrate +sqlalchemy-utils WebOb diff --git a/packages/data/tests/unittest_utils/__init__.py b/packages/data/tests/unittest_utils/__init__.py index 2115ad35783..290bb9ee29b 120000 --- a/packages/data/tests/unittest_utils/__init__.py +++ b/packages/data/tests/unittest_utils/__init__.py @@ -1 +1 @@ -../../test/unit/unittest_utils/__init__.py \ No newline at end of file +../../../../test/unit/unittest_utils/__init__.py \ No newline at end of file diff --git a/packages/data/tests/unittest_utils/tempfilecache.py b/packages/data/tests/unittest_utils/tempfilecache.py index 93fb83f1df7..588f6eec7d8 120000 --- a/packages/data/tests/unittest_utils/tempfilecache.py +++ b/packages/data/tests/unittest_utils/tempfilecache.py @@ -1 +1 @@ -../../test/unit/unittest_utils/tempfilecache.py \ No newline at end of file +../../../../test/unit/unittest_utils/tempfilecache.py \ No newline at end of file diff --git a/packages/data/tests/unittest_utils/utility.py b/packages/data/tests/unittest_utils/utility.py index 54dd4ca7a3b..3c69a8fd67e 120000 --- a/packages/data/tests/unittest_utils/utility.py +++ b/packages/data/tests/unittest_utils/utility.py @@ -1 +1 @@ -../../test/unit/unittest_utils/utility.py \ No newline at end of file +../../../../test/unit/unittest_utils/utility.py \ No newline at end of file diff --git a/packages/meta/HISTORY.rst b/packages/meta/HISTORY.rst new file mode 100644 index 00000000000..ec2f0e746e9 --- /dev/null +++ b/packages/meta/HISTORY.rst @@ -0,0 +1,12 @@ +.. :changelog: + +History +------- + +.. to_doc + +--------------------- +19.9.0.dev0 +--------------------- + +* Initial import from dev branch of Galaxy during 19.09 development cycle. diff --git a/packages/meta/LICENSE b/packages/meta/LICENSE new file mode 120000 index 00000000000..1ef648f64b3 --- /dev/null +++ b/packages/meta/LICENSE @@ -0,0 +1 @@ +../../LICENSE.txt \ No newline at end of file diff --git a/packages/meta/MANIFEST.in b/packages/meta/MANIFEST.in new file mode 100644 index 00000000000..6072b22831e --- /dev/null +++ b/packages/meta/MANIFEST.in @@ -0,0 +1,4 @@ +recursive-include lib/galaxy/web/framework/templates *.mako +recursive-include lib/galaxy/web/framework/static * +recursive-include lib/galaxy/tools/bundled * +recursive-include lib/galaxy/web/proxy/js * diff --git a/packages/meta/Makefile b/packages/meta/Makefile new file mode 120000 index 00000000000..37af8bae5ba --- /dev/null +++ b/packages/meta/Makefile @@ -0,0 +1 @@ +../package.Makefile \ No newline at end of file diff --git a/packages/meta/README.rst b/packages/meta/README.rst new file mode 120000 index 00000000000..c768ff7d970 --- /dev/null +++ b/packages/meta/README.rst @@ -0,0 +1 @@ +../../README.rst \ No newline at end of file diff --git a/packages/meta/dev-requirements.txt b/packages/meta/dev-requirements.txt new file mode 120000 index 00000000000..467b90d7a23 --- /dev/null +++ b/packages/meta/dev-requirements.txt @@ -0,0 +1 @@ +../package-dev-requirements.txt \ No newline at end of file diff --git a/packages/meta/galaxy/project_galaxy_meta.py b/packages/meta/galaxy/project_galaxy_meta.py new file mode 100644 index 00000000000..96958e8d653 --- /dev/null +++ b/packages/meta/galaxy/project_galaxy_meta.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- + +__version__ = '19.9.0.dev0' + +PROJECT_NAME = "galaxy" +PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" +PROJECT_URL = "https://github.com/galaxyproject/galaxy" +PROJECT_AUTHOR = 'Galaxy Project and Community' +PROJECT_DESCRIPTION = 'Galaxy Server Metapackage' +PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' +RAW_CONTENT_URL = "https://raw.github.com/%s/%s/master/" % ( + PROJECT_USERAME, PROJECT_NAME +) diff --git a/packages/meta/scripts b/packages/meta/scripts new file mode 120000 index 00000000000..9aec9dc5a06 --- /dev/null +++ b/packages/meta/scripts @@ -0,0 +1 @@ +../build_scripts \ No newline at end of file diff --git a/packages/meta/setup.cfg b/packages/meta/setup.cfg new file mode 120000 index 00000000000..eb7cf09393f --- /dev/null +++ b/packages/meta/setup.cfg @@ -0,0 +1 @@ +../../setup.cfg \ No newline at end of file diff --git a/packages/meta/setup.py b/packages/meta/setup.py new file mode 100644 index 00000000000..ba0cd02ba7b --- /dev/null +++ b/packages/meta/setup.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import ast +import os +import re +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +SOURCE_DIR = "galaxy" + +_version_re = re.compile(r'__version__\s+=\s+(.*)') + +with open('%s/project_galaxy_meta.py' % SOURCE_DIR, 'rb') as f: + init_contents = f.read().decode('utf-8') + + def get_var(var_name): + pattern = re.compile(r'%s\s+=\s+(.*)' % var_name) + match = pattern.search(init_contents).group(1) + return str(ast.literal_eval(match)) + + version = get_var("__version__") + PROJECT_NAME = get_var("PROJECT_NAME") + PROJECT_URL = get_var("PROJECT_URL") + PROJECT_AUTHOR = get_var("PROJECT_AUTHOR") + PROJECT_EMAIL = get_var("PROJECT_EMAIL") + PROJECT_DESCRIPTION = get_var("PROJECT_DESCRIPTION") + +TEST_DIR = 'tests' +PACKAGES = [] +ENTRY_POINTS = ''' + [console_scripts] + galaxy-paster=galaxy.util.pastescript.serve:run + galaxy-main=galaxy.main:main + galaxy-config=galaxy.config.script:main + galaxy-manage-db=galaxy.model.orm.scripts:manage_db +''' +PACKAGE_DATA = { + # Be sure to update MANIFEST.in for source dist. +} +PACKAGE_DIR = { + SOURCE_DIR: SOURCE_DIR, +} + +readme = open('README.rst').read() +history = open('HISTORY.rst').read().replace('.. :changelog:', '') + +if os.path.exists("requirements.txt"): + requirements = open("requirements.txt").read().split("\n") +else: + # In tox, it will cover them anyway. + requirements = [] + + +test_requirements = [ + # TODO: put package test requirements here +] + + +# TODO: +# package_data={ +# 'galaxy': [ +# 'config/sample/*', +# 'exceptions/error_codes.json', +# 'datatypes/converters/*.xml', +# 'datatypes/display_applications/configs/*/*.xml', +# 'datatypes/set_metadata_tool.xml', +# 'jobs/runners/util/job_script/CLUSTER_SLOTS_STATEMENT.sh', +# 'jobs/runners/util/job_script/MEMORY_STATEMENT.sh', +# 'jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh', +# 'tools/imp_exp/imp_history_from_archive.xml', +# 'tools/imp_exp/exp_history_to_archive.xml', +# 'tools/data_fetch.xml', +# 'model/migrate/migrate.cfg', +# 'dependencies/*.txt', +# 'util/docutils_template.txt', +# ], +# 'tool_shed': [ +# 'galaxy_install/migrate/migrate.cfg', +# 'galaxy_install/migrate/scripts/*', +# 'scripts/bootstrap_tool_shed/parse_run_sh_args.sh', +# 'scripts/bootstrap_tool_shed/bootstrap_tool_shed.sh', +# 'scripts/bootstrap_tool_shed/user_info.xml', +# ], +# }, +# package_dir={'': 'lib'}, +# include_package_data=True, + + +setup( + name=PROJECT_NAME, + version=version, + description=PROJECT_DESCRIPTION, + long_description=readme + '\n\n' + history, + long_description_content_type='text/x-rst', + author=PROJECT_AUTHOR, + author_email=PROJECT_EMAIL, + url=PROJECT_URL, + packages=PACKAGES, + entry_points=ENTRY_POINTS, + package_data=PACKAGE_DATA, + package_dir=PACKAGE_DIR, + include_package_data=True, + install_requires=requirements, + extras_require={ + 'postgresql': ['psycopg2-binary'], + }, + license="AFL", + zip_safe=False, + keywords='galaxy', + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Intended Audience :: Science/Research', + 'Environment :: Console', + 'License :: OSI Approved :: Academic Free License (AFL)', + 'Operating System :: POSIX', + 'Topic :: Scientific/Engineering :: Bio-Informatics', + 'Topic :: Software Development', + 'Topic :: Software Development :: Code Generators', + 'Topic :: Software Development :: Testing', + 'Natural Language :: English', + "Programming Language :: Python :: 2", + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + ], + test_suite=TEST_DIR, + tests_require=test_requirements +) diff --git a/packages/test.sh b/packages/test.sh index aaf70bd4464..fda2337bf50 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -2,6 +2,10 @@ set -e +# Don't display the pip progress bar when running under CI +PIP_PROGRESS_BAR= +[ "$CI" = "true" ] && PIP_PROGRESS_BAR='--progress-bar off' + # Change to packages directory. cd "$(dirname "$0")" @@ -11,7 +15,7 @@ TEST_ENV_DIR=${TEST_ENV_DIR:-$(mktemp -d -t gxpkgtestenvXXXXXX)} virtualenv -p "$TEST_PYTHON" "$TEST_ENV_DIR" . "${TEST_ENV_DIR}/bin/activate" -pip install pytest +pip install ${PIP_PROGRESS_BAR} pytest # ensure ordered by dependency dag PACKAGE_DIRS=( @@ -31,17 +35,15 @@ PACKAGE_DIRS=( ) # containers has no tests, tool_util not yet working 100%, # data has many problems quota, tool shed install database, etc.. -RUN_TESTS=(1 1 1 0 1 0 1 0 1 0 0 0 0) - - +RUN_TESTS=(1 1 1 1 1 1 1 1 1 0 0 0 0) for ((i=0; i<${#PACKAGE_DIRS[@]}; i++)); do package_dir=${PACKAGE_DIRS[$i]} run_tests=${RUN_TESTS[$i]} cd "$package_dir" - pip install -e . + pip install ${PIP_PROGRESS_BAR} -e . if [ "$package_dir" = "util" ]; then - pip install -e '.[template,jstree]' + pip install ${PIP_PROGRESS_BAR} -e '.[template,jstree]' fi if [[ "$run_tests" == "1" ]]; then diff --git a/packages/util/requirements.txt b/packages/util/requirements.txt index e15e5db251e..aad3e2dc97a 100644 --- a/packages/util/requirements.txt +++ b/packages/util/requirements.txt @@ -5,4 +5,5 @@ docutils markupsafe packaging pyyaml +routes six>=1.9.0 diff --git a/run_tests.sh b/run_tests.sh index 08322c750a5..884015751a9 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -363,7 +363,7 @@ do -a|-api|--api) GALAXY_TEST_USE_HIERARCHICAL_OBJECT_STORE="True" # Run these tests with a non-trivial object store. export GALAXY_TEST_USE_HIERARCHICAL_OBJECT_STORE - GALAXY_TEST_TOOL_CONF="config/tool_conf.xml.sample,test/functional/tools/samples_tool_conf.xml" + GALAXY_TEST_TOOL_CONF="lib/galaxy/config/sample/tool_conf.xml.sample,test/functional/tools/samples_tool_conf.xml" test_script="pytest" report_file="./run_api_tests.html" if [ $# -gt 1 ]; then @@ -375,7 +375,7 @@ do fi ;; -selenium|--selenium) - GALAXY_TEST_TOOL_CONF="config/tool_conf.xml.sample,test/functional/tools/samples_tool_conf.xml" + GALAXY_TEST_TOOL_CONF="lib/galaxy/config/sample/tool_conf.xml.sample,test/functional/tools/samples_tool_conf.xml" test_script="./scripts/functional_tests.py" report_file="./run_selenium_tests.html" skip_client_build="" @@ -430,7 +430,7 @@ do shift 1 ;; -main|-main_tools|--main_tools) - GALAXY_TEST_TOOL_CONF="config/tool_conf.xml.sample,config/tool_conf.xml.main" + GALAXY_TEST_TOOL_CONF="lib/galaxy/config/sample/tool_conf.xml.sample,config/tool_conf.xml.main" marker="-m tool" test_script="pytest" report_file="run_framework_tests.html" @@ -507,7 +507,7 @@ do -u|-unit|--unit) report_file="run_unit_tests.html" test_script="pytest" - unit_extra='--doctest-modules --ignore lib/galaxy/web/proxy/js/node_modules/ --ignore lib/galaxy/webapps/tool_shed/controllers --ignore lib/galaxy/jobs/runners/chronos.py --ignore lib/galaxy/webapps/tool_shed/model/migrate' + unit_extra='--doctest-modules --ignore lib/galaxy/web/proxy/js/node_modules/ --ignore lib/galaxy/webapps/tool_shed/controllers --ignore lib/galaxy/jobs/runners/chronos.py --ignore lib/galaxy/webapps/tool_shed/model/migrate --ignore lib/galaxy/tools/bundled' if [ $# -gt 1 ]; then unit_extra="$unit_extra $2" shift 2 @@ -517,7 +517,7 @@ do fi ;; -i|-integration|--integration) - GALAXY_TEST_TOOL_CONF="config/tool_conf.xml.sample,test/functional/tools/samples_tool_conf.xml" + GALAXY_TEST_TOOL_CONF="lib/galaxy/config/sample/tool_conf.xml.sample,test/functional/tools/samples_tool_conf.xml" test_script="pytest" report_file="./run_integration_tests.html" if [ $# -gt 1 ]; then diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index 25d421bb795..1c85bb8cb3a 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -33,10 +33,6 @@ for arg in "$@"; do done SAMPLES=" - config/migrated_tools_conf.xml.sample - config/shed_tool_conf.xml.sample - config/shed_tool_data_table_conf.xml.sample - config/shed_data_manager_conf.xml.sample lib/tool_shed/scripts/bootstrap_tool_shed/user_info.xml.sample tool-data/shared/ucsc/builds.txt.sample tool-data/shared/ucsc/manual_builds.txt.sample @@ -191,15 +187,18 @@ if [ $DEV_WHEELS -eq 1 ]; then requirement_args="$requirement_args -r ${GALAXY_DEV_REQUIREMENTS}" fi +PIP_PROGRESS_BAR= +[ "$CI" = "true" ] && PIP_PROGRESS_BAR='--progress-bar off' + if [ $FETCH_WHEELS -eq 1 ]; then - pip install $requirement_args --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}" + pip install $requirement_args --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}" ${PIP_PROGRESS_BAR} GALAXY_CONDITIONAL_DEPENDENCIES=$(PYTHONPATH=lib python -c "from __future__ import print_function; import galaxy.dependencies; print('\n'.join(galaxy.dependencies.optional('$GALAXY_CONFIG_FILE')))") if [ -n "$GALAXY_CONDITIONAL_DEPENDENCIES" ]; then if pip list --format=columns | grep "psycopg2[\(\ ]*2.7.3" > /dev/null; then echo "An older version of psycopg2 (non-binary, version 2.7.3) has been detected. Galaxy now uses psycopg2-binary, which will be installed after removing psycopg2." pip uninstall -y psycopg2 psycopg2-binary fi - echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}" + echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}" ${PIP_PROGRESS_BAR} fi fi diff --git a/scripts/galaxy b/scripts/galaxy new file mode 100755 index 00000000000..323875c78fd --- /dev/null +++ b/scripts/galaxy @@ -0,0 +1,126 @@ +#!/bin/bash + +show_help() { +cat <&2 + exit 1 + fi + ;; + -c|--config) + if [ $# -gt 1 ]; then + GALAXY_CONFIG_DIR=$2 + shift 2 + else + echo "--config requires explicit argument" 1>&2 + exit 1 + fi + ;; + --) + shift + break + ;; + *) + break; + ;; + esac +done + +SCRIPTS_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PROJECT_DIRECTORY=$SCRIPTS_DIRECTORY/.. + +GALAXY_CONFIG_DIR=${GALAXY_CONFIG_DIR:-"./config"} +export GALAXY_CONFIG_DIR + +GALAXY_LOCAL_ENV=${GALAXY_LOCAL_ENV:-$GALAXY_CONFIG_DIR/local_env.sh} +export GALAXY_LOCAL_ENV + +GALAXY_VIRTUALENV=${GALAXY_VIRTUALENV:-$PROJECT_DIRECTORY/.venv} +export GALAXY_VIRTUALENV + +if [ -e $GALAXY_LOCAL_ENV ]; +then + echo "Sourcing file $GALAXY_LOCAL_ENV" + . $GALAXY_LOCAL_ENV +fi + +if [ -d $GALAXY_VIRTUALENV ]; +then + echo "Activating virtual environment $GALAXY_VIRTUALENV/bin/activate" + . $GALAXY_VIRTUALENV/bin/activate +fi + +GALAXY_CONFIG_SAMPLE_FILE="${PROJECT_DIRECTORY}/galaxy.ini.sample" +if [ -z "$GALAXY_CONFIG_FILE" ]; then + if [ -f "${GALAXY_CONFIG_DIR}/galaxy.ini" ]; then + GALAXY_CONFIG_FILE="${GALAXY_CONFIG_DIR}/galaxy.ini" + else + GALAXY_CONFIG_FILE="$GALAXY_CONFIG_SAMPLE_FILE" + fi + export GALAXY_CONFIG_FILE +fi + +if [ -z "$MODE" ]; +then + if hash uwsgi 2>/dev/null; then + MODE="uwsgi" + else + MODE="paster" + fi +fi + +if [ "$MODE" == "uwsgi" ]; then + echo "Starting galaxy with command [uwsgi --ini-paste \"$GALAXY_CONFIG_FILE\" \"$@\"]" + uwsgi --ini-paste "$GALAXY_CONFIG_FILE" "$@" +elif [ "$MODE" == "paster" ]; then + echo "Starting galaxy with command [galaxy-paster serve \"$GALAXY_CONFIG_FILE\" \"$@\"]" + galaxy-paster serve "$GALAXY_CONFIG_FILE" "$@" +elif [ "$MODE" == "webless" ]; then + if hash galaxy-main 2>/dev/null; then + # Running from pip install since + # galaxy-main is available on PATH. + echo "Starting galaxy with command [galaxy-main]" + galaxy-main + else + # Running locally. + PROJECT_DIRECTORY=$GALAXY_SCRIPTS_DIR/.. + cd $PROJECT_DIRECTORY + export PYTHONPATH=.:$PYTHONPATH + echo "Starting galaxy with command [python lib/galaxy/main.py]" + python lib/pulsar/main.py + fi +else + echo "Unknown mode passed to --mode argument." 1>&2 + exit 1 +fi diff --git a/scripts/galaxy-main b/scripts/galaxy-main index c51ae7f6329..1d649f2ffad 100755 --- a/scripts/galaxy-main +++ b/scripts/galaxy-main @@ -47,12 +47,16 @@ log = logging.getLogger(__name__) real_file = os.path.realpath(__file__) GALAXY_ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(real_file), os.pardir)) -GALAXY_LIB_DIR = os.path.join(GALAXY_ROOT_DIR, "lib") -try: - sys.path.insert(1, GALAXY_LIB_DIR) -except Exception: - log.exception("Failed to add Galaxy to sys.path") - raise +if not os.path.exists(os.path.join(GALAXY_ROOT_DIR, 'run.sh')): + # Galaxy is installed + GALAXY_ROOT_DIR = None +else: + GALAXY_LIB_DIR = os.path.join(GALAXY_ROOT_DIR, "lib") + try: + sys.path.insert(1, GALAXY_LIB_DIR) + except Exception: + log.exception("Failed to add Galaxy to sys.path") + raise from galaxy.util import unicodify from galaxy.web_stack import get_app_kwds @@ -130,9 +134,10 @@ def register_signals(): def app_loop(args, log): try: config_builder = GalaxyConfigBuilder(args) + config_env = GALAXY_ROOT_DIR is not None galaxy_app = load_galaxy_app( config_builder, - config_env=True, + config_env=config_env, log=log, attach_to_pools=args.attach_to_pool, ) @@ -162,6 +167,9 @@ def find_config(supplied_config, galaxy_root): if supplied_config: return supplied_config + if galaxy_root is None: + return os.path.abspath('galaxy.yml') + # If not explicitly supplied an config, check galaxy.ini and then # just resort to sample if that has not been configured. for guess in DEFAULT_INIS: @@ -202,7 +210,7 @@ class GalaxyConfigBuilder(object): arg_parser.add_argument("-c", "--config-file", default=None, help="Galaxy config file (defaults to config/galaxy.ini)") arg_parser.add_argument("--ini-path", default=None, help="DEPRECATED: use -c/--config-file") arg_parser.add_argument("--app", default=None, help="app section in config file (defaults to 'galaxy' for YAML/JSON, 'main' (w/ 'app:' prepended) for INI") - arg_parser.add_argument("-d", "--daemonize", default=False, help="Daemonzie process", action="store_true") + arg_parser.add_argument("-d", "--daemonize", default=False, help="Daemonize process", action="store_true") arg_parser.add_argument("--daemon-log-file", default=None, help="log file for daemon script ") arg_parser.add_argument("--log-file", default=None, help="Galaxy log file (overrides log configuration in config_file if set)") arg_parser.add_argument("--pid-file", default=DEFAULT_PID, help="pid file (default is %s)" % DEFAULT_PID) diff --git a/scripts/manage_tool_dependencies.py b/scripts/manage_tool_dependencies.py index 57175f4837f..72dba77e92f 100644 --- a/scripts/manage_tool_dependencies.py +++ b/scripts/manage_tool_dependencies.py @@ -3,12 +3,9 @@ import sys sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib'))) -from galaxy.config import ( - configure_logging, - find_path, - find_root, -) +from galaxy.config import configure_logging from galaxy.tool_util.deps import build_dependency_manager +from galaxy.util.properties import find_config_file from galaxy.util.script import main_factory DESCRIPTION = "Script to manage tool dependencies (with focus on a Conda environments)." @@ -26,8 +23,9 @@ def _build_dependency_manager_no_config(kwargs): which we do not have available in this script (an optimization). """ configure_logging(kwargs) - root = find_root(kwargs) - dependency_resolvers_config_file = find_path(kwargs, "dependency_resolvers_config_file", root) + base, ext = os.path.splitext(kwargs.get('dependency_resolvers_config_file', 'dependency_resolvers_conf.xml')) + dependency_resolvers_config_file = find_config_file(base, exts=[ext.lstrip('.')]) + # FIXME: default is wrong for installed Galaxy dependency_manager = build_dependency_manager(app_config_dict=kwargs, conf_file=dependency_resolvers_config_file, default_tool_dependency_dir="database/dependencies") return dependency_manager diff --git a/scripts/migrate_tools/0002_tools.sh b/scripts/migrate_tools/0002_tools.sh deleted file mode 100644 index 7262196b5a9..00000000000 --- a/scripts/migrate_tools/0002_tools.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -cd "$(dirname "$0")"/../.. -python ./scripts/migrate_tools/migrate_tools.py 0002_tools.xml "$@" diff --git a/scripts/migrate_tools/0003_tools.sh b/scripts/migrate_tools/0003_tools.sh deleted file mode 100644 index f8a1ba8bb16..00000000000 --- a/scripts/migrate_tools/0003_tools.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -cd "$(dirname "$0")"/../.. -python ./scripts/migrate_tools/migrate_tools.py 0003_tools.xml "$@" diff --git a/scripts/migrate_tools/0004_tools.sh b/scripts/migrate_tools/0004_tools.sh deleted file mode 100644 index 7ae608f3683..00000000000 --- a/scripts/migrate_tools/0004_tools.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -cd "$(dirname "$0")"/../.. -python ./scripts/migrate_tools/migrate_tools.py 0004_tools.xml "$@" diff --git a/scripts/migrate_tools/0005_tools.sh b/scripts/migrate_tools/0005_tools.sh deleted file mode 100644 index da7f9c09098..00000000000 --- a/scripts/migrate_tools/0005_tools.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -cd "$(dirname "$0")"/../.. -python ./scripts/migrate_tools/migrate_tools.py 0005_tools.xml "$@" diff --git a/scripts/migrate_tools/0006_tools.sh b/scripts/migrate_tools/0006_tools.sh deleted file mode 100644 index ff85522a5f9..00000000000 --- a/scripts/migrate_tools/0006_tools.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -cd "$(dirname "$0")"/../.. -python ./scripts/migrate_tools/migrate_tools.py 0006_tools.xml "$@" diff --git a/scripts/migrate_tools/0007_tools.sh b/scripts/migrate_tools/0007_tools.sh deleted file mode 100644 index 8a88f44420c..00000000000 --- a/scripts/migrate_tools/0007_tools.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -cd "$(dirname "$0")"/../.. -python ./scripts/migrate_tools/migrate_tools.py 0007_tools.xml "$@" diff --git a/scripts/migrate_tools/0008_tools.sh b/scripts/migrate_tools/0008_tools.sh deleted file mode 100644 index d312237023b..00000000000 --- a/scripts/migrate_tools/0008_tools.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -cd "$(dirname "$0")"/../.. -python ./scripts/migrate_tools/migrate_tools.py 0008_tools.xml "$@" diff --git a/scripts/migrate_tools/0009_tools.sh b/scripts/migrate_tools/0009_tools.sh deleted file mode 100644 index 78242fe2422..00000000000 --- a/scripts/migrate_tools/0009_tools.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -cd "$(dirname "$0")"/../.. -python ./scripts/migrate_tools/migrate_tools.py 0009_tools.xml "$@" diff --git a/scripts/migrate_tools/0010_tools.sh b/scripts/migrate_tools/0010_tools.sh deleted file mode 100644 index 03c65dd9973..00000000000 --- a/scripts/migrate_tools/0010_tools.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -cd "$(dirname "$0")"/../.. -python ./scripts/migrate_tools/migrate_tools.py 0010_tools.xml "$@" diff --git a/scripts/migrate_tools/0011_tools.sh b/scripts/migrate_tools/0011_tools.sh deleted file mode 100644 index e06a6be40b3..00000000000 --- a/scripts/migrate_tools/0011_tools.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -cd "$(dirname "$0")"/../.. -python ./scripts/migrate_tools/migrate_tools.py 0011_tools.xml "$@" diff --git a/scripts/migrate_tools/0012_tools.sh b/scripts/migrate_tools/0012_tools.sh deleted file mode 100644 index 2df312f980a..00000000000 --- a/scripts/migrate_tools/0012_tools.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -cd "$(dirname "$0")"/../.. -python ./scripts/migrate_tools/migrate_tools.py 0012_tools.xml "$@" diff --git a/scripts/paster.py b/scripts/paster.py index bcd2080724c..90fb608d940 100755 --- a/scripts/paster.py +++ b/scripts/paster.py @@ -20,8 +20,4 @@ try: except Exception: sys.exit(1) -if 'LOG_TEMPFILES' in os.environ: - from log_tempfile import TempFile - _log_tempfile = TempFile() - serve.run() diff --git a/templates/webapps/galaxy/admin/tool_sheds.mako b/templates/webapps/galaxy/admin/tool_sheds.mako deleted file mode 100644 index 68f3f116864..00000000000 --- a/templates/webapps/galaxy/admin/tool_sheds.mako +++ /dev/null @@ -1,33 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> - -<%def name="title()">Configured Galaxy tool sheds - -<%def name="stylesheets()"> - ${parent.stylesheets()} - ${h.css( "library" )} - - -%if message: - ${render_msg( message, status )} -%endif - -
-
Accessible Galaxy tool sheds
-
- <% shed_id = 0 %> - %for name, url in trans.app.tool_shed_registry.tool_sheds.items(): - <% margin_cls = "mt-2" if shed_id > 0 else "" %> - - -
- <% shed_id += 1 %> - %endfor -
-
diff --git a/templates/webapps/galaxy/workflow/editor.mako b/templates/webapps/galaxy/workflow/editor.mako index 09f66de636d..187f634ca99 100644 --- a/templates/webapps/galaxy/workflow/editor.mako +++ b/templates/webapps/galaxy/workflow/editor.mako @@ -91,13 +91,15 @@ %else:
%endif - %if "[[" in tool.description and "]]" in tool.description: - ${tool.description.replace( '[[', '' % tool.id ).replace( "]]", "" )} - %elif tool.name: - ${tool.name} ${tool.description} - %else: - ${tool.description} - %endif + + %if "[[" in tool.description and "]]" in tool.description: + ${tool.description.replace( '[[', '' % tool.id ).replace( "]]", "" )} + %elif tool.name: + ${tool.name} ${tool.description} + %else: + ${tool.description} + %endif +
%endif %endif @@ -146,10 +148,10 @@
- diff --git a/test/api/test_libraries.py b/test/api/test_libraries.py index a9f8d5f4f05..11454c23695 100644 --- a/test/api/test_libraries.py +++ b/test/api/test_libraries.py @@ -232,6 +232,26 @@ class LibrariesApiTestCase(api.ApiTestCase, TestsDatasets): self._assert_status_code_is(create_response, 200) self._assert_has_keys(create_response.json(), "name", "id") + def test_create_dataset_in_subfolder(self): + library = self.library_populator.new_private_library("ForCreateDatasets") + folder_response = self._create_folder(library) + self._assert_status_code_is(folder_response, 200) + folder_id = folder_response.json()[0]['id'] + subfolder_response = self._create_subfolder(folder_id) + self._assert_status_code_is(folder_response, 200) + print(subfolder_response.json()) + subfolder_id = subfolder_response.json()['id'] + history_id = self.dataset_populator.new_history() + hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3 sub")['id'] + payload = {'from_hda_id': hda_id} + create_response = self._post("folders/%s/contents" % subfolder_id, payload) + self._assert_status_code_is(create_response, 200) + self._assert_has_keys(create_response.json(), "name", "id") + dataset_update_time = create_response.json()['update_time'] + container_fetch_response = self.galaxy_interactor.get("folders/%s/contents" % folder_id) + container_update_time = container_fetch_response.json()['folder_contents'][0]['update_time'] + assert dataset_update_time == container_update_time, container_fetch_response + def test_update_dataset_in_folder(self): ld = self._create_dataset_in_folder_in_library("ForUpdateDataset") data = {'name': 'updated_name', 'file_ext': 'fastq', 'misc_info': 'updated_info', 'genome_build': 'updated_genome_build'} @@ -301,6 +321,13 @@ class LibrariesApiTestCase(api.ApiTestCase, TestsDatasets): ) return self._post("libraries/%s/contents" % library["id"], data=create_data) + def _create_subfolder(self, containing_folder_id): + create_data = dict( + description="new subfolder desc", + name="New Subfolder", + ) + return self._post("folders/%s" % containing_folder_id, data=create_data) + def _create_dataset_in_folder_in_library(self, library_name): library = self.library_populator.new_private_library(library_name) folder_response = self._create_folder(library) diff --git a/test/base/driver_util.py b/test/base/driver_util.py index 920684a211a..92587939904 100644 --- a/test/base/driver_util.py +++ b/test/base/driver_util.py @@ -178,11 +178,13 @@ def setup_galaxy_config( if tool_dependency_dir is None: tool_dependency_dir = tempfile.mkdtemp(dir=tmpdir, prefix="tool_dependencies") tool_data_table_config_path = _tool_data_table_config_path(default_tool_data_table_config_path) - default_data_manager_config = 'config/data_manager_conf.xml.sample' + default_data_manager_config = None for data_manager_config in ['config/data_manager_conf.xml', 'data_manager_conf.xml']: if os.path.exists(data_manager_config): default_data_manager_config = data_manager_config - data_manager_config_file = "%s,test/functional/tools/sample_data_manager_conf.xml" % default_data_manager_config + data_manager_config_file = "test/functional/tools/sample_data_manager_conf.xml" + if default_data_manager_config is not None: + data_manager_config_file = "%s,%s" % (default_data_manager_config, data_manager_config_file) master_api_key = get_master_api_key() cleanup_job = 'never' if ("GALAXY_TEST_NO_CLEANUP" in os.environ or "TOOL_SHED_TEST_NO_CLEANUP" in os.environ) else 'onsuccess' @@ -203,8 +205,6 @@ def setup_galaxy_config( tool_conf = "%s,%s" % (tool_conf, shed_tool_conf) shed_tool_data_table_config = default_shed_tool_data_table_config - if shed_tool_data_table_config is None: - shed_tool_data_table_config = 'config/shed_tool_data_table_conf.xml' config = dict( admin_users='test@bx.psu.edu', @@ -300,7 +300,7 @@ def _tool_data_table_config_path(default_tool_data_table_config_path=None): if tool_data_table_config_path is None: # ... otherise find whatever Galaxy would use as the default and # the sample data for fucntional tests to that. - default_tool_data_config = 'config/tool_data_table_conf.xml.sample' + default_tool_data_config = 'lib/galaxy/config/sample/tool_data_table_conf.xml.sample' for tool_data_config in ['config/tool_data_table_conf.xml', 'tool_data_table_conf.xml']: if os.path.exists(tool_data_config): default_tool_data_config = tool_data_config @@ -567,7 +567,7 @@ def build_galaxy_app(simple_kwargs): """ log.info("Galaxy database connection: %s", simple_kwargs["database_connection"]) simple_kwargs['global_conf'] = get_webapp_global_conf() - simple_kwargs['global_conf']['__file__'] = "config/galaxy.yml.sample" + simple_kwargs['global_conf']['__file__'] = "lib/galaxy/config/sample/galaxy.yml.sample" simple_kwargs = load_app_properties( kwds=simple_kwargs ) diff --git a/test/functional/tools/upload.py b/test/functional/tools/upload.py index 6cf43a566b3..282421f9585 120000 --- a/test/functional/tools/upload.py +++ b/test/functional/tools/upload.py @@ -1 +1 @@ -../../../tools/data_source/upload.py \ No newline at end of file +../../../lib/galaxy/tools/bundled/data_source/upload.py \ No newline at end of file diff --git a/test/functional/tools/upload.xml b/test/functional/tools/upload.xml index 3096f6a2d4c..684a91b47ee 120000 --- a/test/functional/tools/upload.xml +++ b/test/functional/tools/upload.xml @@ -1 +1 @@ -../../../tools/data_source/upload.xml \ No newline at end of file +../../../lib/galaxy/tools/bundled/data_source/upload.xml \ No newline at end of file diff --git a/test/integration/test_datatype_upload.py b/test/integration/test_datatype_upload.py index 58e45b0bb55..46275580351 100644 --- a/test/integration/test_datatype_upload.py +++ b/test/integration/test_datatype_upload.py @@ -18,7 +18,7 @@ SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) TEST_FILE_DIR = '%s/../../lib/galaxy/datatypes/test' % SCRIPT_DIRECTORY TestData = collections.namedtuple('UploadDatatypesData', 'path datatype uploadable') GALAXY_ROOT = os.path.abspath('%s/../../' % SCRIPT_DIRECTORY) -DATATYPES_CONFIG = os.path.join(GALAXY_ROOT, 'config/datatypes_conf.xml.sample') +DATATYPES_CONFIG = os.path.join(GALAXY_ROOT, 'lib/galaxy/config/sample/datatypes_conf.xml.sample') PARENT_SNIFFER_MAP = {'fastqsolexa': 'fastq'} diff --git a/test/integration/test_local_job_cancellation.py b/test/integration/test_local_job_cancellation.py index abcaa0a1576..d01cea8bd2e 100644 --- a/test/integration/test_local_job_cancellation.py +++ b/test/integration/test_local_job_cancellation.py @@ -18,29 +18,53 @@ class LocalJobCancellationTestCase(integration_util.IntegrationTestCase): super(LocalJobCancellationTestCase, self).setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) + def setup_cat_data_and_sleep(self, history_id): + hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3") + running_inputs = { + "input1": {"src": "hda", "id": hda1["id"]}, + "sleep_time": 240, + } + running_response = self.dataset_populator.run_tool( + "cat_data_and_sleep", + running_inputs, + history_id, + assert_ok=False, + ).json() + job_dict = running_response["jobs"][0] + return job_dict + + def test_cancel_job_with_admin_message(self): + with self.dataset_populator.test_history() as history_id: + job_dict = self.setup_cat_data_and_sleep(history_id) + self.galaxy_interactor.wait_for(lambda: self._get("jobs/%s" % job_dict['id']).json()['state'] != 'running', + what="Wait for job to start running", + maxseconds=60) + app = self._app + sa_session = app.model.context.current + Job = app.model.Job + job = sa_session.query(Job).filter_by(tool_id="cat_data_and_sleep").order_by(Job.create_time.desc()).first() + # This is how the admin controller code cancels a job + job.job_stderr = 'admin cancelled job' + job.set_state(app.model.Job.states.DELETED_NEW) + sa_session.add(job) + sa_session.flush() + self.galaxy_interactor.wait_for(lambda: self._get("jobs/%s" % job_dict['id']).json()['state'] != 'error', + what="Wait for job to end in error", + maxseconds=60) + def test_kill_process(self): """ """ with self.dataset_populator.test_history() as history_id: - hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3") - running_inputs = { - "input1": {"src": "hda", "id": hda1["id"]}, - "sleep_time": 240, - } - running_response = self.dataset_populator.run_tool( - "cat_data_and_sleep", - running_inputs, - history_id, - assert_ok=False, - ).json() - job_dict = running_response["jobs"][0] + job_dict = self.setup_cat_data_and_sleep(history_id) app = self._app sa_session = app.model.context.current external_id = None state = False + Job = app.model.Job - job = sa_session.query(app.model.Job).filter_by(tool_id="cat_data_and_sleep").one() + job = sa_session.query(Job).filter_by(tool_id="cat_data_and_sleep").order_by(Job.create_time.desc()).first() # Not checking the state here allows the change from queued to running to overwrite # the change from queued to deleted_new in the API thread - this is a problem because # the job will still run. See issue https://github.com/galaxyproject/galaxy/issues/4960. diff --git a/test/shed_functional/base/twilltestcase.py b/test/shed_functional/base/twilltestcase.py index 33d2f80e87f..67d936106e3 100644 --- a/test/shed_functional/base/twilltestcase.py +++ b/test/shed_functional/base/twilltestcase.py @@ -521,10 +521,8 @@ class ShedTwillTestCase(FunctionalTestCase): self.check_for_strings(strings_displayed, strings_not_displayed) def browse_tool_shed(self, url, strings_displayed=None, strings_not_displayed=None): - params = { - 'tool_shed_url': url - } - self.visit_galaxy_url('/admin_toolshed/browse_tool_shed', params=params) + url = '/repository/browse_valid_categories' + self.visit_url(url) self.check_for_strings(strings_displayed, strings_not_displayed) def browse_tool_dependencies(self, strings_displayed=None, strings_not_displayed=None): diff --git a/test/shed_functional/functional/test_1000_install_basic_repository.py b/test/shed_functional/functional/test_1000_install_basic_repository.py index 4fc6d2223db..e44f61a2fcf 100644 --- a/test/shed_functional/functional/test_1000_install_basic_repository.py +++ b/test/shed_functional/functional/test_1000_install_basic_repository.py @@ -71,8 +71,6 @@ class BasicToolShedFeatures(ShedTwillTestCase): def test_0010_browse_tool_sheds(self): """Browse the available tool sheds in this Galaxy instance.""" self.galaxy_login(email=common.admin_email, username=common.admin_username) - self.visit_galaxy_url('/admin_toolshed/browse_tool_sheds') - self.check_page_for_string('Embedded tool shed for functional tests') self.browse_tool_shed(url=self.url, strings_displayed=['Test 0000 Basic Repository Features 1', 'Test 0000 Basic Repository Features 2']) def test_0015_browse_test_0000_category(self): diff --git a/test/unit/datatypes/test_sniff.py b/test/unit/datatypes/test_sniff.py index faf75443281..3217ae6aaf6 100644 --- a/test/unit/datatypes/test_sniff.py +++ b/test/unit/datatypes/test_sniff.py @@ -18,11 +18,10 @@ def assert_converts_to_1234_convert_sep2tabs(content, expected='1\t2\n3\t4\n'): def assert_converts_to_1234_convert(content, block_size=1024): - fname = get_test_fname('temp2.txt') - with open(fname, 'w') as fh: - fh.write(content) - rval = convert_newlines(fname, tmp_prefix="gxtest", tmp_dir=tempfile.gettempdir(), block_size=block_size) - actual_contents = open(fname).read() + with tempfile.NamedTemporaryFile(delete=False, mode='w') as tf: + tf.write(content) + rval = convert_newlines(tf.name, tmp_prefix="gxtest", tmp_dir=tempfile.gettempdir(), block_size=block_size) + actual_contents = open(tf.name).read() assert '1 2\n3 4\n' == actual_contents, actual_contents assert rval == (2, None), "rval != %s for %s" % (rval, content) diff --git a/test/unit/jobs/test_job_configuration.py b/test/unit/jobs/test_job_configuration.py index 9aec01cb6d4..33acdf18607 100644 --- a/test/unit/jobs/test_job_configuration.py +++ b/test/unit/jobs/test_job_configuration.py @@ -13,8 +13,8 @@ from galaxy.web_stack import ApplicationStack, UWSGIApplicationStack # File would be slightly more readable if contents were embedded directly, but # there are advantages to testing the documentation/examples. -SIMPLE_JOB_CONF = os.path.join(os.path.dirname(__file__), "..", "..", "..", "config", "job_conf.xml.sample_basic") -ADVANCED_JOB_CONF = os.path.join(os.path.dirname(__file__), "..", "..", "..", "config", "job_conf.xml.sample_advanced") +SIMPLE_JOB_CONF = os.path.join(os.path.dirname(__file__), "..", "..", "..", "lib", "galaxy", "config", "sample", "job_conf.xml.sample_basic") +ADVANCED_JOB_CONF = os.path.join(os.path.dirname(__file__), "..", "..", "..", "lib", "galaxy", "config", "sample", "job_conf.xml.sample_advanced") ADVANCED_JOB_CONF_YAML = os.path.join(os.path.dirname(__file__), "job_conf.sample_advanced.yml") CONDITIONAL_RUNNER_JOB_CONF = os.path.join(os.path.dirname(__file__), "conditional_runners_job_conf.xml") HANDLER_TEMPLATE_JOB_CONF = os.path.join(os.path.dirname(__file__), "handler_template_job_conf.xml") diff --git a/test/unit/queue_worker/conftest.py b/test/unit/queue_worker/conftest.py index 603265ebcc8..2e867eec1a4 100644 --- a/test/unit/queue_worker/conftest.py +++ b/test/unit/queue_worker/conftest.py @@ -3,6 +3,11 @@ import tempfile import pytest +try: + import psycopg2 +except ImportError: + psycopg2 = None + from galaxy.util import which from ..unittest_utils import galaxy_mock @@ -57,6 +62,8 @@ def database_app(request): if request.param == 'postgres_app': if not which('initdb'): pytest.skip("initdb must be on PATH for postgresql fixture") + if not psycopg2: + pytest.skip("psycopg2 must be installed for postgresql fixture") if request.param == 'sqlite_rabbitmq_app': if not os.environ.get('GALAXY_TEST_AMQP_INTERNAL_CONNECTION'): pytest.skip("rabbitmq tests will be skipped if GALAXY_TEST_AMQP_INTERNAL_CONNECTION env var is unset") diff --git a/test/unit/test_auth.py b/test/unit/test_auth.py new file mode 100644 index 00000000000..557de59cb13 --- /dev/null +++ b/test/unit/test_auth.py @@ -0,0 +1,18 @@ +from galaxy.auth.providers.alwaysreject import AlwaysReject +from galaxy.auth.providers.localdb import LocalDB +from galaxy.model import User + + +def test_alwaysreject(): + t = AlwaysReject() + assert t.authenticate('testmail', 'testuser', 'secret', dict()) == (None, '', '') + + +def test_localdb(): + user = User(email='testmail', username='tester') + user.set_password_cleartext('test') + t = LocalDB() + reject = t.authenticate_user(user, 'wrong', {'redact_username_in_logs': False}) + accept = t.authenticate_user(user, 'test', {'redact_username_in_logs': False}) + assert reject is False + assert accept is True diff --git a/test/unit/test_containers.py b/test/unit/test_containers.py new file mode 100644 index 00000000000..02e30a462e0 --- /dev/null +++ b/test/unit/test_containers.py @@ -0,0 +1,8 @@ +import galaxy.containers + + +def test_dummy(): + t = galaxy.containers.parse_containers_config('') + assert t == {'_default_': {'type': 'docker'}} + s = galaxy.containers.docker_model.DockerAttributeContainer() + assert s.members == frozenset([]) diff --git a/test/unit/tools/test_toolbox.py b/test/unit/tools/test_toolbox.py index f16b565e693..e592f8e43d2 100644 --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -9,11 +9,11 @@ import routes from six import string_types from galaxy import model +from galaxy.config_watchers import ConfigWatchers from galaxy.model import tool_shed_install from galaxy.model.tool_shed_install import mapping from galaxy.tools import ToolBox from galaxy.tools.cache import ToolCache -from galaxy.webapps.galaxy.config_watchers import ConfigWatchers from .test_tool_loader import ( SIMPLE_MACRO, SIMPLE_TOOL_WITH_MACRO diff --git a/test/unit/unittest_utils/galaxy_mock.py b/test/unit/unittest_utils/galaxy_mock.py index b1254494041..6feaab340ba 100644 --- a/test/unit/unittest_utils/galaxy_mock.py +++ b/test/unit/unittest_utils/galaxy_mock.py @@ -113,6 +113,7 @@ class MockAppConfig(Bunch): self.security = idencoding.IdEncodingHelper(id_secret='6e46ed6483a833c100e68cc3f1d0dd76') self.database_connection = kwargs.get('database_connection', "sqlite:///:memory:") self.use_remote_user = kwargs.get('use_remote_user', False) + self.data_dir = '/tmp' self.file_path = '/tmp' self.jobs_directory = '/tmp' self.new_file_path = '/tmp' @@ -155,6 +156,12 @@ class MockAppConfig(Bunch): def config_dict(self): return self.dict() + def __getattr__(self, name): + # Handle the automatic config file _set options + if name.endswith('_file_set'): + return False + return super(MockAppConfig, self).__getattr__(name) + class MockWebapp(object): diff --git a/tox.ini b/tox.ini index 0aec466cf97..f06e5e66ab7 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,7 @@ commands = lint: bash .ci/flake8_wrapper.sh unit: bash run_tests.sh -u whitelist_externals = bash +passenv = CI setenv = py{35,36,37}-first_startup: GALAXY_VIRTUAL_ENV=.venv3 unit: GALAXY_VIRTUAL_ENV={envdir}