diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml new file mode 100644 index 00000000000..3a3e5437c63 --- /dev/null +++ b/.github/workflows/integration.yaml @@ -0,0 +1,48 @@ +name: Integration +on: [push, pull_request] +env: + GALAXY_TEST_DBURI: 'postgres://postgres:postgres@localhost:5432/galaxy?client_encoding=utf8' +jobs: + + test: + name: Test + runs-on: ubuntu-18.04 + strategy: + matrix: + python-version: [3.7] + subset: ['upload_datatype', 'extended_metadata', 'kubernetes', 'not (upload_datatype or extended_metadata or kubernetes)'] + services: + postgres: + image: postgres:11 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + steps: + - name: Setup Minikube + if: matrix.subset == 'kubernetes' + id: minikube + uses: CodingNagger/minikube-setup-action@v1.0.2 + - name: Launch Minikube + if: matrix.subset == 'kubernetes' + run: eval ${{ steps.minikube.outputs.launcher }} + - name: Check pods + if: matrix.subset == 'kubernetes' + run: | + kubectl get pods + - uses: actions/checkout@v1 + with: + fetch-depth: 1 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Cache venv dir + uses: actions/cache@v1 + id: cache-venv-galaxy + with: + path: .venv + key: cache-venv-galaxy-${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }} + - name: Run tests + run: './run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"' diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 830f2384304..800d4762594 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -124,7 +124,7 @@ The following individuals have contributed code to Galaxy: * Chinmay Rao * Matt Ralston * ramezrawas -* Eric Rasche +* Helena Rasche * Athos Ribeiro * Andrew Robinson * Devon Ryan diff --git a/client/galaxy/scripts/components/FavoritesButton.vue b/client/galaxy/scripts/components/FavoritesButton.vue index fefffc206ac..e3a3b695f92 100644 --- a/client/galaxy/scripts/components/FavoritesButton.vue +++ b/client/galaxy/scripts/components/FavoritesButton.vue @@ -8,15 +8,20 @@ role="button" aria-label="Show favorite tools" > - + + + diff --git a/client/galaxy/scripts/components/admin/JobLock.vue b/client/galaxy/scripts/components/admin/JobLock.vue new file mode 100644 index 00000000000..d613b8236b6 --- /dev/null +++ b/client/galaxy/scripts/components/admin/JobLock.vue @@ -0,0 +1,45 @@ + + + diff --git a/client/galaxy/scripts/components/admin/Jobs.vue b/client/galaxy/scripts/components/admin/Jobs.vue index 3f9ace9f7ab..5ed69a8bea2 100644 --- a/client/galaxy/scripts/components/admin/Jobs.vue +++ b/client/galaxy/scripts/components/admin/Jobs.vue @@ -1,8 +1,17 @@ @@ -170,12 +163,7 @@ @@ -191,9 +179,16 @@ import { getAppRoot } from "onload/loadConfig"; import UtcDate from "components/UtcDate"; import axios from "axios"; +import JobDetails from "./JobDetails"; +import JobLock from "./JobLock"; + +function cancelJob(jobId, message) { + const url = `${getAppRoot()}api/jobs/${jobId}`; + return axios.delete(url, { data: { message: message } }); +} export default { - components: { UtcDate }, + components: { UtcDate, JobDetails, JobLock }, data() { return { jobsItems: [], @@ -204,15 +199,11 @@ export default { { key: "user" }, { key: "tool_id", label: "Tool", tdClass: ["break-word"] }, { key: "state" }, - { key: "input_dataset", label: "Inputs" }, { key: "job_runner_name", label: "Job Runner" }, { key: "job_runner_external_id", label: "PID/Cluster ID", sortable: true } ], - showCommandLine: false, cutoff: 180, cutoffDisplay: 180, - jobLock: false, - jobLockDisplay: false, selectedStopJobIds: [], selectedJobId: null, allSelected: false, @@ -226,17 +217,6 @@ export default { }; }, watch: { - jobLock(newVal) { - axios - .get(`${getAppRoot()}admin/jobs_control?job_lock=${this.jobLock}`) - .then(response => { - this.jobLock = response.data.job_lock; - this.jobLockDisplay = response.data.job_lock; - }) - .catch(error => { - console.error(error); - }); - }, selectedStopJobIds(newVal) { if (newVal.length === 0) { this.indeterminate = false; @@ -255,8 +235,6 @@ export default { this.busy = true; let params = []; params.push(`cutoff=${this.cutoff}`); - params.push(`stop_msg=${this.stopMessage}`); - params.push(`stop=${this.selectedStopJobIds.join()}`); params = params.join("&"); axios .get(`${getAppRoot()}admin/jobs_list?${params}`) @@ -266,8 +244,6 @@ export default { this.cutoffDisplay = response.data.cutoff; this.message = response.data.message; this.status = response.data.status; - this.jobLock = response.data.job_lock; - this.jobLockDisplay = response.data.job_lock; this.loading = false; this.busy = false; }) @@ -281,9 +257,11 @@ export default { this.update(); }, onStopJobs() { - this.update(); - this.selectedStopJobIds = []; - this.stopMessage = ""; + axios.all(this.selectedStopJobIds.map(jobId => cancelJob(jobId, this.stopMessage))).then(res => { + this.update(); + this.selectedStopJobIds = []; + this.stopMessage = ""; + }); }, clickJobInfo(id) { this.selectedJobId = id; @@ -319,18 +297,10 @@ export default { }, computeFields(fields) { const f = Array.from(fields).slice(0); - if (this.showCommandLine) { - f.splice(6, 0, { - key: "command_line", - tdClass: ["text-white", "bg-dark", "break-word"] - }); - } return f; }, toggleAll(checked) { - this.selectedStopJobIds = checked - ? this.jobsItemsModel.reduce((acc, j) => [...acc, j["job_info"]["id"]], []) - : []; + this.selectedStopJobIds = checked ? this.jobsItemsModel.reduce((acc, j) => [...acc, j["id"]], []) : []; }, galaxyKwdToBootstrap(status) { let variant = "info"; diff --git a/client/galaxy/scripts/components/login/Login.vue b/client/galaxy/scripts/components/login/Login.vue index 396a97de72b..f2410c9cddf 100644 --- a/client/galaxy/scripts/components/login/Login.vue +++ b/client/galaxy/scripts/components/login/Login.vue @@ -40,11 +40,13 @@
- + + + - Sign in with + Sign in with {{ idp.charAt(0).toUpperCase() + idp.slice(1) }} @@ -78,20 +80,11 @@ export default { }, data() { const galaxy = getGalaxyInstance(); - const oidc_idps = galaxy.config.oidc; // Icons to use for each IdP const oidc_idps_icons = { google: "https://developers.google.com/identity/images/btn_google_signin_light_normal_web.png", elixir: "https://elixir-europe.org/sites/default/files/images/login-button-orange.png" }; - // Add default icons to IdPs without icons - oidc_idps - .filter(function(key) { - return oidc_idps_icons[key] === undefined; - }) - .forEach(function(idp) { - oidc_idps_icons[idp] = "fa fa-id-card"; - }); return { login: null, password: null, @@ -103,7 +96,7 @@ export default { redirect: galaxy.params.redirect, session_csrf_token: galaxy.session_csrf_token, enable_oidc: galaxy.config.enable_oidc, - oidc_idps: oidc_idps, + oidc_idps: galaxy.config.oidc, oidc_idps_icons: oidc_idps_icons }; }, diff --git a/client/galaxy/scripts/layout/menu.js b/client/galaxy/scripts/layout/menu.js index ae26743f9c4..2e63e123a8e 100644 --- a/client/galaxy/scripts/layout/menu.js +++ b/client/galaxy/scripts/layout/menu.js @@ -31,7 +31,7 @@ function logoutClick() { if (response.data && response.data.redirect_uri) { window.top.location.href = response.data.redirect_uri; } else { - window.top.location.href = galaxy.root; + window.top.location.href = `${galaxy.root}login?logout=true`; } }); } diff --git a/client/galaxy/style/scss/base.scss b/client/galaxy/style/scss/base.scss index f9ac0766b5e..e35d739d64b 100644 --- a/client/galaxy/style/scss/base.scss +++ b/client/galaxy/style/scss/base.scss @@ -8,8 +8,16 @@ @import "overrides.scss"; // Fontawesome configuration. -$fa-font-path: "../../../node_modules/font-awesome/fonts/"; -@import "../../../node_modules/font-awesome/scss/font-awesome.scss"; +//$fa-font-path: "../../../node_modules/font-awesome/fonts/"; +//@import "../../../node_modules/font-awesome/scss/font-awesome.scss"; + +$fa-font-path: "../../../node_modules/@fortawesome/fontawesome-free/webfonts/"; +@import "../../../node_modules/@fortawesome/fontawesome-free/scss/fontawesome"; +@import "../../../node_modules/@fortawesome/fontawesome-free/scss/brands"; +@import "../../../node_modules/@fortawesome/fontawesome-free/scss/regular"; +@import "../../../node_modules/@fortawesome/fontawesome-free/scss/solid"; +@import "../../../node_modules/@fortawesome/fontawesome-free/scss/v4-shims"; +@import "../../../node_modules/@fortawesome/fontawesome-free/scss/_variables"; @import "select2.scss"; @import "jquery.rating.scss"; diff --git a/client/galaxy/style/scss/dataset.scss b/client/galaxy/style/scss/dataset.scss index 1c57cbe23a9..1f27021246f 100644 --- a/client/galaxy/style/scss/dataset.scss +++ b/client/galaxy/style/scss/dataset.scss @@ -1,5 +1,6 @@ @import "sprite-history-states.scss"; @import "sprite-history-buttons.scss"; +@import "~@fortawesome/fontawesome-free/scss/_variables"; // ---------------------------------------------------------------------------- dataset states //.dataset .state-icon { @@ -72,7 +73,7 @@ transform: translate(0px, -16px); } } - content: $fa-var-arrow-up; + content: fa-content($fa-var-arrow-up); } } @@ -83,7 +84,7 @@ font-size: 20px; line-height: 16px; color: red; - content: $fa-var-times-circle; + content: fa-content($fa-var-times-circle); } } @@ -117,7 +118,7 @@ background: $state-default-bg; .state-icon { &:before { - content: $fa-var-clock-o; + content: fa-content($fa-var-clock); } } } @@ -134,7 +135,7 @@ background: $state-paused-bg; .state-icon { &:before { - content: $fa-var-pause; + content: fa-content($fa-var-pause); } } } @@ -143,7 +144,7 @@ background: $state-default-bg; .state-icon { &:before { - content: $fa-var-trash-o; + content: fa-content($fa-var-trash); } } } @@ -156,7 +157,7 @@ .state-icon { font-size: 20px; &:before { - content: $fa-var-lock; + content: fa-content($fa-var-lock); } } } @@ -165,7 +166,7 @@ background: $state-default-bg; .state-icon { &:before { - content: $fa-var-clock-o; + content: fa-content($fa-var-clock); } } } diff --git a/client/galaxy/style/scss/mixins.scss b/client/galaxy/style/scss/mixins.scss index 81f7b907670..233c938d394 100644 --- a/client/galaxy/style/scss/mixins.scss +++ b/client/galaxy/style/scss/mixins.scss @@ -1,6 +1,9 @@ -$fa-font-path: "~font-awesome/fonts/"; -@import "~font-awesome/scss/font-awesome.scss"; -@import "~font-awesome/scss/_mixins"; +//$fa-font-path: "~font-awesome/fonts/"; +//@import "~font-awesome/scss/font-awesome.scss"; +//@import "~font-awesome/scss/_mixins"; +$fa-font-path: "../../../node_modules/@fortawesome/fontawesome-free/webfonts/"; +@import "~@fortawesome/fontawesome-free/scss/fontawesome"; +@import "~@fortawesome/fontawesome-free/scss/_variables"; // Utility mixin expands to container edges @mixin fill() { diff --git a/client/galaxy/style/scss/ui/icon-btn.scss b/client/galaxy/style/scss/ui/icon-btn.scss index 0bafffa2bd0..c3e3e7355fd 100644 --- a/client/galaxy/style/scss/ui/icon-btn.scss +++ b/client/galaxy/style/scss/ui/icon-btn.scss @@ -8,7 +8,6 @@ $icon-btn-border: $btn-default-border; display: inline-block; height: $icon-btn-size; width: $icon-btn-size; - font-size: 125%; // center the icon text-align: center; diff --git a/client/package.json b/client/package.json index 9161e57aca2..757bf64c9aa 100644 --- a/client/package.json +++ b/client/package.json @@ -12,6 +12,11 @@ "license": "AFL-3.0", "dependencies": { "@babel/polyfill": "^7.4.0", + "@fortawesome/fontawesome-free": "^5.11.2", + "@fortawesome/fontawesome-svg-core": "^1.2.25", + "@fortawesome/free-regular-svg-icons": "^5.12.0", + "@fortawesome/free-solid-svg-icons": "^5.11.2", + "@fortawesome/vue-fontawesome": "^0.1.8", "@handsontable/vue": "^2.0.0-beta1", "@johmun/vue-tags-input": "^2.0.1", "@vue/test-utils": "1.0.0-beta.29", @@ -24,7 +29,6 @@ "d3": "3", "decode-uri-component": "^0.2.0", "flush-promises": "^1.0.2", - "font-awesome": "^4.7.0", "glob": "^7.1.4", "handsontable": "^2.0.0", "imask": "^4.1.5", diff --git a/client/yarn.lock b/client/yarn.lock index eb875cefffd..50ad9b7ff3f 100644 --- a/client/yarn.lock +++ b/client/yarn.lock @@ -826,6 +826,47 @@ lodash "^4.17.13" to-fast-properties "^2.0.0" +"@fortawesome/fontawesome-common-types@^0.2.25": + version "0.2.25" + resolved "https://registry.yarnpkg.com/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-0.2.25.tgz#6df015905081f2762e5cfddeb7a20d2e9b16c786" + integrity sha512-3RuZPDuuPELd7RXtUqTCfed14fcny9UiPOkdr2i+cYxBoTOfQgxcDoq77fHiiHcgWuo1LoBUpvGxFF1H/y7s3Q== + +"@fortawesome/fontawesome-common-types@^0.2.26": + version "0.2.26" + resolved "https://registry.yarnpkg.com/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-0.2.26.tgz#6e0b13a752676036f8196f8a1500d53a27b4adc1" + integrity sha512-CcM/fIFwZlRdiWG/25xE/wHbtyUuCtqoCTrr6BsWw7hH072fR++n4L56KPydAr3ANgMJMjT8v83ZFIsDc7kE+A== + +"@fortawesome/fontawesome-free@^5.11.2": + version "5.11.2" + resolved "https://registry.yarnpkg.com/@fortawesome/fontawesome-free/-/fontawesome-free-5.11.2.tgz#8644bc25b19475779a7b7c1fc104bc0a794f4465" + integrity sha512-XiUPoS79r1G7PcpnNtq85TJ7inJWe0v+b5oZJZKb0pGHNIV6+UiNeQWiFGmuQ0aj7GEhnD/v9iqxIsjuRKtEnQ== + +"@fortawesome/fontawesome-svg-core@^1.2.25": + version "1.2.25" + resolved "https://registry.yarnpkg.com/@fortawesome/fontawesome-svg-core/-/fontawesome-svg-core-1.2.25.tgz#24b03391d14f0c6171e8cad7057c687b74049790" + integrity sha512-MotKnn53JKqbkLQiwcZSBJVYtTgIKFbh7B8+kd05TSnfKYPFmjKKI59o2fpz5t0Hzl35vVGU6+N4twoOpZUrqA== + dependencies: + "@fortawesome/fontawesome-common-types" "^0.2.25" + +"@fortawesome/free-regular-svg-icons@^5.12.0": + version "5.12.0" + resolved "https://registry.yarnpkg.com/@fortawesome/free-regular-svg-icons/-/free-regular-svg-icons-5.12.0.tgz#9b7ed5dbd7581aab47960802cc116fdc8b6c8b7d" + integrity sha512-FAvpmylTs0PosHwHrWPQX6/7ODc9M11kCE6AOAujFufDYzqTj2cPHT4yJO7zTEkKdAbbusJzbWpnOboMuyjeQA== + dependencies: + "@fortawesome/fontawesome-common-types" "^0.2.26" + +"@fortawesome/free-solid-svg-icons@^5.11.2": + version "5.11.2" + resolved "https://registry.yarnpkg.com/@fortawesome/free-solid-svg-icons/-/free-solid-svg-icons-5.11.2.tgz#2f2f1459743a27902b76655a0d0bc5ec4d945631" + integrity sha512-zBue4i0PAZJUXOmLBBvM7L0O7wmsDC8dFv9IhpW5QL4kT9xhhVUsYg/LX1+5KaukWq4/cbDcKT+RT1aRe543sg== + dependencies: + "@fortawesome/fontawesome-common-types" "^0.2.25" + +"@fortawesome/vue-fontawesome@^0.1.8": + version "0.1.8" + resolved "https://registry.yarnpkg.com/@fortawesome/vue-fontawesome/-/vue-fontawesome-0.1.8.tgz#615347c56d285eb634315f59fe4f4156e4eee8b4" + integrity sha512-SdFiUD+vFDA/xKuEbnQTVrK8FDxoV0eyQaiHxmCcjAc0+vQe0Kf6oGm28opNPIt8MTgKWR3+Yg3xXP455Ae4tQ== + "@handsontable/vue@^2.0.0-beta1": version "2.0.0" resolved "https://registry.yarnpkg.com/@handsontable/vue/-/vue-2.0.0.tgz#28ae7d247a89738088abc32584562925dde18db0" @@ -5917,11 +5958,6 @@ follow-redirects@^1.0.0: dependencies: debug "=3.1.0" -font-awesome@^4.7.0: - version "4.7.0" - resolved "https://registry.yarnpkg.com/font-awesome/-/font-awesome-4.7.0.tgz#8fa8cf0411a1a31afd07b06d2902bb9fc815a133" - integrity sha1-j6jPBBGhoxr9B7BtKQK7n8gVoTM= - for-in@^0.1.3: version "0.1.8" resolved "https://registry.yarnpkg.com/for-in/-/for-in-0.1.8.tgz#d8773908e31256109952b1fdb9b3fa867d2775e1" diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index e244b0a8480..337c64051bc 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -3622,6 +3622,120 @@ :Type: str +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``enable_beta_markdown_export`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Enable export of Galaxy Markdown documents (pages and workflow + reports) to PDF. Requires manual installation and setup of + weasyprint (latest version available for Python 2.7 is 0.42). +:Default: ``false`` +:Type: bool + + +~~~~~~~~~~~~~~~~~~~~~~~ +``markdown_export_css`` +~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + CSS file to apply to all Markdown exports to PDF - currently used + by WeasyPrint during rendering an HTML export of the document to + PDF. +:Default: ``markdown_export.css`` +:Type: str + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``markdown_export_css_pages`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + CSS file to apply to "Galaxy Page" exports to PDF. Generally + prefer markdown_export_css, but this is here for deployments that + would like to tailor different kinds of exports. +:Default: ``markdown_export_pages.css`` +:Type: str + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``markdown_export_css_invocation_reports`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + CSS file to apply to invocation report exports to PDF. Generally + prefer markdown_export_css, but this is here for deployments that + would like to tailor different kinds of exports. +:Default: ``markdown_export_invocation_reports.css`` +:Type: str + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``markdown_export_prologue`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Prologue Markdown/HTML to apply to markdown exports to PDF. + Allowing branded headers. +:Default: ```` +:Type: str + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``markdown_export_epilogue`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Prologue Markdown/HTML to apply to markdown exports to PDF. + Allowing branded footers. +:Default: ```` +:Type: str + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``markdown_export_prologue_pages`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Alternative to markdown_export_prologue that applies just to page + exports. +:Default: ```` +:Type: str + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``markdown_export_prologue_invocation_reports`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Alternative to markdown_export_prologue that applies just to + invocation report exports. +:Default: ```` +:Type: str + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``markdown_export_epilogue_pages`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Alternative to markdown_export_epilogue that applies just to page + exports. +:Default: ```` +:Type: str + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``markdown_export_epilogue_invocation_reports`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Alternative to markdown_export_epilogue that applies just to + invocation report exports. +:Default: ```` +:Type: str + + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``job_resource_params_file`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/source/admin/special_topics/interactive_environments.svg b/doc/source/admin/special_topics/interactive_environments.svg index 88f5de92c8d..55c81e4c55f 100644 --- a/doc/source/admin/special_topics/interactive_environments.svg +++ b/doc/source/admin/special_topics/interactive_environments.svg @@ -484,7 +484,7 @@ Galaxy Jupyter Graphic - Björn Grüning, Torsten Houwaart, Eric Rasche + Björn Grüning, Torsten Houwaart, Helena Rasche en_US diff --git a/doc/source/api/guidelines.rst b/doc/source/api/guidelines.rst index c49f94f8ce1..4260144da2f 100644 --- a/doc/source/api/guidelines.rst +++ b/doc/source/api/guidelines.rst @@ -14,14 +14,14 @@ these guidelines. should properly set the status code according to the result - in particular 2XX responses should be used for successful requests, 4XX for various kinds of client errors, and 5XX for the errors on the server side. - - If there is an error processing some part of request (one item in a list + - If there is an error processing some part of the request (one item in a list for instance), the status code should be set to reflect the error and the partial result may or may not be returned depending on the controller - this behavior should be documented. - API methods should throw a finite number of exceptions (defined in :mod:`galaxy.exceptions`) and these should subclass `MessageException` and not paste/wsgi HTTP exceptions. When possible, - the framework itself should be responsible catching these exceptions, + the framework itself should be responsible for catching these exceptions, setting the status code, and building an error response. - Error responses should not consist of plain text strings - they should be dictionaries describing the error and containing the following:: @@ -53,7 +53,7 @@ consider them when modifying the API. changes and so a second opinion on API changes should be sought. - New API functionality should include functional tests. These functional tests should be implemented in Python and placed in - `test/functional/api`. + `lib/galaxy_test/api`. - Changes to reflect modifications to the API should be pushed upstream to the BioBlend project if possible. diff --git a/doc/source/dev/interactive_environments.rst b/doc/source/dev/interactive_environments.rst index 27b7925032b..99d8f190f60 100644 --- a/doc/source/dev/interactive_environments.rst +++ b/doc/source/dev/interactive_environments.rst @@ -545,4 +545,4 @@ If everything went well, at this point you should see a directory listing show u .. image:: interactive_environments_success.png If you find yourself encountering difficulties, the "Hello, World" IE is -available in a `GitHub repo `__, and there are people on the IRC channel who can help debug. +available in a `GitHub repo `__, and there are people on the IRC channel who can help debug. diff --git a/doc/source/releases/15.05.rst b/doc/source/releases/15.05.rst index 5de92f86cfc..7877b56a6c6 100644 --- a/doc/source/releases/15.05.rst +++ b/doc/source/releases/15.05.rst @@ -55,10 +55,10 @@ Enhancements `Pull Request 143`_ * Add NCBI SRA datatype - thanks to Matt Shirley. `Pull Request 87`_ * Stronger toolbox filtering. `Pull Request 119`_ -* Allow updating Tool Shed repositories via the API - thanks to Eric Rasche. +* Allow updating Tool Shed repositories via the API - thanks to Helena Rasche. `Pull Request 30`_ * Expose category list in show call for Tool Shed repositories - thanks to - Eric Rasche. `Pull Request 29`_ + Helena Rasche. `Pull Request 29`_ * Add API endpoint to create Tool Shed repositories. `Pull Request 2`_ * Do not configure Galaxy to use the test Tool Shed by default. `Pull Request 38`_ @@ -67,7 +67,7 @@ Enhancements * Enhance multi-selection widgets to allow key combinations ``Ctrl-A`` and ``Ctrl-X``. e8564d7_, `Trello `__ * New, consistent button for displaying citation BibTeX. `Pull Request 19`_ -* Improved ``README`` reflecting move to Github - thanks in part to Eric +* Improved ``README`` reflecting move to Github - thanks in part to Helena Rasche. `PR #2 (old repo) `__, 226e826_, 2650d09_, 7d5dde8_ @@ -87,14 +87,14 @@ Enhancements * Preserve spaces in ``Count1`` tool (``tools/filters/uniq.xml``) - thanks to Peter Cock. `Pull Request 13`_ * `Interactive Environments`_ improvements and fixes from multiple - developers including Eric Rasche and Björn Grüning. `Pull Request 69`_, + developers including Helena Rasche and Björn Grüning. `Pull Request 69`_, `Pull Request 73`_, `Pull Request 131`_, `Pull Request 135`_, `Pull Request 152`_, `Pull Request 197`_ -* Enable multi-part upload for exporting files with the GenomeSpace export +* Enable multi-part upload for exporting files with the GenomeSpace export tool. `Pull Request 74`_, `Trello `__ * Large refactoring, expansion, and increase in test coverage for "managers". `Pull Request 76`_ -* Improved display of headers in tool help. 157eba6_, +* Improved display of headers in tool help. 157eba6_, `Biostar `__ * Uniform configuration of "From" field for sent emails - thanks to Nicola Soranzo. `Pull Request 23`_ @@ -112,8 +112,8 @@ Enhancements * Add test tool demonstrating citing a Github repository. 65def71_ * Add option to track all automated changes to the integrated tool panel. 10bb492_ -* Make tool version explicit in all distribution tool - thanks to Peter Cock. - `Pull Request 14`_. +* Make tool version explicit in all distribution tool - thanks to Peter Cock. + `Pull Request 14`_. * Relocate the external metadata setting script. `Pull Request 7`_ * Parameterize script used to pull new builds from the UCSC Browser. e4e5df0_ @@ -141,7 +141,7 @@ Fixes properly. `Pull Request 198`_, `Trello `__ * Fix scatter plot API communications for certain proxied Galaxy instances - thanks to @yhoogstrate. `Pull Request 89`_ -* Fix bug in collectl_ job metrics plugin - thanks to Carrie Ganote. +* Fix bug in collectl_ job metrics plugin - thanks to Carrie Ganote. `Pull Request 231`_ * Fix late validation of tool parameters. `Pull Request 115`_ * Fix ``fasta_to_tabular_converter.py`` (for implicit conversion) - thanks to @@ -161,7 +161,7 @@ Fixes * Fix to remove redundant version ids in tool version selector. `Pull Request 244`_ * Fix for downloading metadata files. `Pull Request 234`_ -* Fix for history failing to render if it contains more exotic dataset +* Fix for history failing to render if it contains more exotic dataset collection types. `Pull Request 196`_ * Fixes for BaseURLToolParameter. `Pull Request 247`_ * Fix to suppress pysam binary incompatibility warning when using datatypes diff --git a/doc/source/releases/15.07.rst b/doc/source/releases/15.07.rst index b27322668f0..2c6d7676a61 100644 --- a/doc/source/releases/15.07.rst +++ b/doc/source/releases/15.07.rst @@ -12,7 +12,7 @@ Enhancements .. enhancements * Interactive environment (IE) framework updates including a new IE for RStudio. - (Thanks to a team effort spearheaded by `@erasche `__.) + (Thanks to a team effort spearheaded by `@hexylena `__.) `Pull Request 229`_, `Pull Request 230`_ * Rework workflow editor to use newer tool form components and allow construction of workflows with tools explicitly producing output collections. @@ -27,8 +27,8 @@ Enhancements focus on the API documentation. (Thanks to `@nsoranzo `__.) `Pull Request 399`_, `Pull Request 405`_ -* Add EDAM IDs to Galaxy datatypes. (Thanks to a team effort initiated by `@erasche - `__.) +* Add EDAM IDs to Galaxy datatypes. (Thanks to a team effort initiated by `@hexylena + `__.) `Pull Request 80`_, `Pull Request 342`_ * Improved reports app sorting and bug fixes thanks to Daniel Bouchard (`@Airistotal `__). Even more @@ -66,11 +66,11 @@ Enhancements `__.) `Pull Request 441`_ * Add emboss datatypes (previously migrated to Tool Shed). (Thanks to `@bgruening `__.) `Pull Request 148`_ -* Add new Hmmer & Stockholm datatypes. (Thanks to `@erasche - `__.) `Pull Request 218`_ -* Add an RData datatype. (Thanks to `@erasche - `__.) `Pull Request 239`_ -* Add mauve XMFA datatype (Thanks to `@erasche `__.) +* Add new Hmmer & Stockholm datatypes. (Thanks to `@hexylena + `__.) `Pull Request 218`_ +* Add an RData datatype. (Thanks to `@hexylena + `__.) `Pull Request 239`_ +* Add mauve XMFA datatype (Thanks to `@hexylena `__.) `Pull Request 270`_ * Add ConnectivityTable (\*.ct) (RNA 2D structure format) datatype. (Thanks to `@yhoogstrate `__.) @@ -114,10 +114,10 @@ Enhancements `Pull Request 186`_ * Switch visualizations registry to entry_point config definition for built-in visualizations. `Pull Request 243`_ -* Set default value for IE sig proxy (Thanks to `@erasche - `__.) `Pull Request 197`_ +* Set default value for IE sig proxy (Thanks to `@hexylena + `__.) `Pull Request 197`_ * Other smaller interactive environment updates. (Thanks to - `@erasche `__.) + `@hexylena `__.) `Pull Request 412`_ * Refuse to load interactive debugging middleware under uwsgi. `Pull Request 200`_ @@ -179,7 +179,7 @@ Enhancements * Add configurable Docker hostname (Thanks to `@bgruening `__.) `Pull Request 356`_ * Improve XML styling of display applications. - (Thanks to `@erasche `__.) + (Thanks to `@hexylena `__.) `Pull Request 364`_ * README improvements including - adding shields for Biostar, IRC, and Travis. `Pull Request 368`_, `Pull Request 369`_ diff --git a/doc/source/releases/15.07_announce.rst b/doc/source/releases/15.07_announce.rst index ca02d6dc81e..ef7e21708ad 100644 --- a/doc/source/releases/15.07_announce.rst +++ b/doc/source/releases/15.07_announce.rst @@ -10,7 +10,7 @@ Highlights **Interactive Environments** The interactive environments (IE) framework features several updates this release - and Galaxy is now distributed with a new IE for RStudio implemented by Eric Rasche. + and Galaxy is now distributed with a new IE for RStudio implemented by Helena Rasche. **Workflow Editor Enhancements** The workflow editor has been significantly revamped - it now uses newer tool form @@ -18,9 +18,9 @@ Highlights output collections. **Policies for Committers and Pull Requests** - The process for adding committers to the project and fielding pull requests has + The process for adding committers to the project and fielding pull requests has been formalized and documented in the source code. Three new committers have been - added to the project - Björn Grüning, Nicola Soranzo, and Eric Rasche. + added to the project - Björn Grüning, Nicola Soranzo, and Helena Rasche. `Github `__ @@ -28,17 +28,17 @@ Highlights New .. code-block:: shell - + % git clone -b master https://github.com/galaxyproject/galaxy.git Update to latest stable release .. code-block:: shell - + % git checkout master && pull --ff-only origin master Update to exact version .. code-block:: shell - + % git checkout v15.07 @@ -47,8 +47,8 @@ Update to exact version Upgrade .. code-block:: shell - - % hg pull + + % hg pull % hg update latest_15.07 diff --git a/doc/source/releases/15.10.rst b/doc/source/releases/15.10.rst index 3ff09495280..5e517291f41 100644 --- a/doc/source/releases/15.10.rst +++ b/doc/source/releases/15.10.rst @@ -32,10 +32,10 @@ Enhancements * Add ``regex_replace`` tool dependency install action. `Pull Request 457`_ * Document Galaxy interactive environments (GIEs). - (Thanks to `@erasche `__.) + (Thanks to `@hexylena `__.) `Pull Request 803`_ -* Explicit project procedure language on reversions and WIP. (Thanks to `@erasche - `__.) +* Explicit project procedure language on reversions and WIP. (Thanks to `@hexylena + `__.) `Pull Request 556`_ * Readme improvemenets - GitHub issues, security email contact and CI testing (Thanks to `@nsoranzo `__.) @@ -97,7 +97,7 @@ Enhancements * Expose tool config file path to admins via the API. `Pull Request 652`_ * Add Galaxy developer documentation section to docs. - (Thanks to `@erasche `__.) + (Thanks to `@hexylena `__.) `Pull Request 654`_ * Password reset improvements. `Pull Request 655`_ @@ -115,7 +115,7 @@ Enhancements * Set a default value for copy history dialog radio input (to copy *all*). `Pull Request 693`_ * Remove various code duplication. - (Thanks to `@erasche `__.) + (Thanks to `@hexylena `__.) `Pull Request 694`_ * Use ``setup.py`` for Python all installations during dependency installation. (Thanks to `@bgruening `__.) @@ -163,13 +163,13 @@ Enhancements `__.) `Pull Request 777`_ * Unify the docker command, command_inject, and command_wrapper (Thanks to - `@erasche `__.) + `@hexylena `__.) `Pull Request 782`_ * Test case to prevent regression of `#753 `__. `Pull Request 784`_ * Refactor interactive environment launching to use Docker's ``-P`` option. - (Thanks to `@erasche `__.) + (Thanks to `@hexylena `__.) `Pull Request 790`_ * Print ``has_metrics`` class name instead of object memory address. (Thanks to `@nsoranzo `__.) @@ -205,7 +205,7 @@ Enhancements `Pull Request 847`_ * Remove unused ``echo`` tool `Pull Request 848`_ -* Update interactive environments for this release. (Thanks to `@erasche `__.) +* Update interactive environments for this release. (Thanks to `@hexylena `__.) `Pull Request 849`_ * Enhancements for IPython interactive environment allowing it run in certain deployments. @@ -296,10 +296,10 @@ Fixes (Thanks to `@nsoranzo `__.) `Pull Request 596`_ * Fix bug in reload API where id wasn't recognised. - (Thanks to `@erasche `__.) + (Thanks to `@hexylena `__.) `Pull Request 619`_ * Correct typo in tool shed action handler code. - (Thanks to `@erasche `__.) + (Thanks to `@hexylena `__.) `Pull Request 561`_ * Small fixes for UI functional tests. `Pull Request 485`_, `Pull Request 487`_ @@ -404,7 +404,7 @@ Fixes * Tool form error messages and rerun fixes. `Pull Request 766`_ * Quote value to fix configparser error in interactive environments. - (Thanks to `@erasche `__.) + (Thanks to `@hexylena `__.) `Pull Request 775`_ * Fix upload event binding. `Pull Request 778`_ @@ -425,7 +425,7 @@ Fixes (Thanks to `@nsoranzo `__.) `Pull Request 838`_ * Expose all API keys to admins under ``REMOTE_USER``. - (Thanks to `@erasche `__.) + (Thanks to `@hexylena `__.) `Pull Request 873`_ * Various fixes for data libraries. `Pull Request 878`_ diff --git a/doc/source/releases/16.01.rst b/doc/source/releases/16.01.rst index 1fd79ee567e..2534f6eb049 100644 --- a/doc/source/releases/16.01.rst +++ b/doc/source/releases/16.01.rst @@ -58,7 +58,7 @@ Enhancements `Pull Request 1405`_ * Add search on username/any for API calls to user and use Select2 for sharing workflows/pages/histories - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 1111`_ * Improve masthead and scratchbook UI and architecture. `Pull Request 1407`_, `Pull Request 1424`_ @@ -113,7 +113,7 @@ Enhancements environment variables. `Pull Request 564`_, `Pull Request 889`_ * Reorganize history menu - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 985`_ * Look up installed tools from different toolshed(s) (thanks to `@mvdbeek `__.) @@ -141,7 +141,7 @@ Enhancements (thanks to `@scholtalbers `__.) `Pull Request 912`_ * Add an API script to upload directory as a data library - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 918`_ * Enable dockerized toolshed tests. `Pull Request 942`_ @@ -197,7 +197,7 @@ Enhancements * Use common exceptions in tools. `Pull Request 874`_ * Refactor the remote user middleware for complexity - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 875`_ * Allowed for capitalized package names such as R in unlinked tool shed dependency resolver @@ -221,7 +221,7 @@ Enhancements * Add Workflow editor UI for step labels. `Pull Request 1251`_ * Ease the grunt uglify process - (Thanks to `@erasche `__.) + (Thanks to `@hexylena `__.) `Pull Request 1238`_ * Add beta run workflow form based on the newer tool form code. `Pull Request 1249`_ @@ -260,7 +260,7 @@ Enhancements * Unify and abstract code for checking if file looks like a tool definition. `Pull Request 1368`_ * Added hashes to pip requirements file - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 1393`_ * Interactive environment updates (thanks to `@bgruening `__.) @@ -376,10 +376,10 @@ Fixes (thanks to `@mvdbeek `__.) `Pull Request 863`_ * Bump version of sqlite3 in nodejs proxy - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 866`_ * Expose API keys to admins under ``REMOTE_USER`` - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 872`_ * Bug fix for tool shed repository API (thanks to `@mvdbeek `__.) @@ -394,7 +394,7 @@ Fixes error. `Pull Request 908`_ * Bug fix for condor runner changes made this release cycle - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 909`_ * Change user preference datatype to text. `Pull Request 916`_ @@ -404,14 +404,14 @@ Fixes * Revise tool URL building. `Pull Request 947`_ * Fix for code checking if a file seems to a tool definition file - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 949`_ * Fix base panels to include scripts as the last element of the body. `Pull Request 969`_ * Grid batch operation fixes. `Pull Request 971`_ * Fix extra files path URL problem. - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 991`_ * Fixes broken env-var declarations for tools with weird chars in their names (thanks to `@yhoogstrate `__.) @@ -485,7 +485,7 @@ Fixes * Fix for creating workflow outputs on initial workflow upload. `Pull Request 1330`_ * Don't query on unencoded IDs for error form - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 1340`_ * If ``GALAXY_SLOTS`` is defined in the environment, use it for the local runner. `Pull Request 1346`_ diff --git a/doc/source/releases/16.04.rst b/doc/source/releases/16.04.rst index c3bc54c31fc..41b68a87d10 100644 --- a/doc/source/releases/16.04.rst +++ b/doc/source/releases/16.04.rst @@ -38,7 +38,7 @@ Enhancements * disable 'hg push' to TS repositories `Pull Request 2033`_ * Generic GIE Launcher, GIE Image Chooser, multiple datasets - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 1403`_ * The entire Python source code is now continuously checked with flake8 for PEP-8 style consistency and common errors. The last 3500 errors were fixed in @@ -49,7 +49,7 @@ Enhancements .. enhancement * Implement new GIE proxy - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 1807`_ * Workflow editor overview click navigation `Pull Request 1843`_ @@ -58,7 +58,7 @@ Enhancements * datalibs: introduce folder management; fix various glitches; refactor `Pull Request 1562`_ * Add makefile target for release process - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 1433`_ * libraries: provide select all/none options when importing `Pull Request 1970`_ @@ -152,7 +152,7 @@ Enhancements * Implement dataset empty input validator. `Pull Request 1808`_ * Makefile help target + formatting - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 1824`_ * Add license and repository information to package.json. `Pull Request 1829`_ @@ -228,7 +228,7 @@ Enhancements * make new beta TS tools installation via API configurable `Pull Request 2088`_ * Patch in visual separation of section parameters - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 2091`_ * Add sample supervisor config (thanks to `@mvdbeek `__.) @@ -447,7 +447,7 @@ Fixes * Fix farbtastic's use of deprecated jquery fns by loading jq-migrate in galaxy.panels.mako . `Pull Request 1972`_ * Remove duplicate help target - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 1980`_ * Fix booleans in workflow editor `Pull Request 1990`_ @@ -477,7 +477,7 @@ Fixes * Show sections in workflow run `Pull Request 2087`_ * Workflow section fix backport - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 2092`_ * Run external local set_metadata jobs in the job's working directory `Pull Request 2094`_ diff --git a/doc/source/releases/16.07.rst b/doc/source/releases/16.07.rst index 911f6d0b029..af7dee0c852 100644 --- a/doc/source/releases/16.07.rst +++ b/doc/source/releases/16.07.rst @@ -35,7 +35,7 @@ Highlighted Enhancements (thanks to `@BalthazarPavot `__.) `Pull Request 2664`_ * Nicer bug report emails - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 2143`_ * Tools with auto_format='true' now connectable in workflow editor (thanks to `@yhoogstrate `__.) @@ -160,7 +160,7 @@ Enhancements * Remove backported importlib and ordereddict. `Pull Request 2380`_ * Note on updating tool testing - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 2381`_ * Docker swarm support for Galaxy Interactive Environments (GIEs) `Pull Request 2386`_, `Pull Request 2392`_ @@ -238,7 +238,7 @@ Enhancements * Allow users to purge datasets by default. `Pull Request 2553`_ * Add Jupyter IE to the allowed visualizations for BAM datasets - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 2572`_ * Python3: port 215 files (thanks to `@nsoranzo `__, `@tshtatland `__, `@xiaoqian1984 `__, `@alanryanwang `__ and `@ericenns `__.) @@ -392,14 +392,14 @@ Fixes (thanks to `@nsoranzo `__.) `Pull Request 2223`_ * Sanitize poorly escaped input value in run workflow form - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 2231`_ * Multi-data collection reductions in repeat blocks are broken. `Pull Request 2233`_ * Encode collection reduce in serializable fashion `Pull Request 2241`_ * Cast everything to a string - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 2252`_ * Use standard model.orm.now for toolshed's password reset token datetime math `Pull Request 2261`_ @@ -426,7 +426,7 @@ Fixes * Build: remove redundant webpack step in makefile `Pull Request 2327`_ * Unicodify has different semantics to str, causing bug reporter bug - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 2336`_ * Wrap statsd timing middleware more closely around the core application `Pull Request 2342`_ @@ -441,7 +441,7 @@ Fixes (thanks to `@bgruening `__.) `Pull Request 2394`_ * Check variable's presence before calling str methods - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 2408`_ * UI, history: correctly handle index API defaults `Pull Request 2413`_ @@ -482,10 +482,10 @@ Fixes * Initialize objectstore for fix_set_dataset_sizes script `Pull Request 2544`_ * Fix old bug in non-allowed_images.yml GIEs - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 2548`_ * More defensive GIE Proxying - (thanks to `@erasche `__.) + (thanks to `@hexylena `__.) `Pull Request 2552`_ * Fixes for showing min/max labels in Trackster. `Pull Request 2555`_ diff --git a/doc/source/releases/16.10.rst b/doc/source/releases/16.10.rst index 7ffb4f35fc1..0d582855502 100644 --- a/doc/source/releases/16.10.rst +++ b/doc/source/releases/16.10.rst @@ -221,7 +221,7 @@ Enhancements (thanks to `@nsoranzo `__). `Pull Request 3033`_ * Remove outdated warning on ID_SECRET - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 3046`_ * Add Galaxy architecture slides documentation to project. `Pull Request 3060`_, `Pull Request 3091`_ @@ -330,10 +330,10 @@ Fixes (thanks to `@sszakony `__). `Pull Request 2808`_ * Allow NodeJS proxy to do reverse proxying - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 2817`_ * Fixes for the RStudio interactive environment - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 2818`_ * Acquire lock before attempting to install conda (thanks to `@mvdbeek `__). diff --git a/doc/source/releases/17.01.rst b/doc/source/releases/17.01.rst index 9b8fbfc723f..d5fb6489487 100644 --- a/doc/source/releases/17.01.rst +++ b/doc/source/releases/17.01.rst @@ -50,20 +50,20 @@ Enhancements `Pull Request 3217`_ * Many improvements to the job information page - including exposing certain job metrics to non-admin users - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 3344`_, `Pull Request 3256`_, `Pull Request 3340`_ * Formalize decision to increase time between releases to four months. The next release will be 17.05. `Pull Request 3271`_ * Add API endpoint to determine who the current user is - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 3423`_ * Extend API version information for Docker-based Galaxy instances (thanks to `@manabuishii `__). `Pull Request 3411`_ * Implement a more complete Genbank datatype with sniffer - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 3339`_ * Add additional image datatypes (thanks to `@ThomasWollmann `__). @@ -114,7 +114,7 @@ Enhancements (thanks to `@bgruening `__). `Pull Request 3328`_ * Allow admins to see API keys under when using remote user authentication from proxies. - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 3338`_ * Limit adjusting Galaxy's Python environment to legacy tools (discussed in Deprecation Notices section of release notes). @@ -149,7 +149,7 @@ Fixes (thanks to `@nsoranzo `__). `Pull Request 3215`_ * Fix user table header in admin view - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 3200`_ * Fix to adjust masthead and tool search. `Pull Request 3055`_ @@ -193,10 +193,10 @@ Fixes (thanks to `@manabuishii `__). `Pull Request 3409`_ * Fix login functionality for RStudio IEs - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 3414`_ * Fix accidental email exposure during history sharing - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 3417`_ * Fix UX problem when sharing objects with users. `Pull Request 3373`_ diff --git a/doc/source/releases/17.05.rst b/doc/source/releases/17.05.rst index 803af07b415..f4aaf560071 100644 --- a/doc/source/releases/17.05.rst +++ b/doc/source/releases/17.05.rst @@ -38,7 +38,7 @@ Enhancements (thanks to `@anuprulez `__ and `@bgruening `__). `Pull Request 3601`_ * Add script to encode or decode the IDs that Galaxy exposes to users and admins - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 3622`_ * Support Docker Engine swarm mode for Interactive Environments. `Pull Request 3666`_ @@ -50,7 +50,7 @@ Enhancements * Container interface abstraction. `Pull Request 3863`_ * Graphite support - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 3964`_ * Normalize the handling of steps and missing tools for the workflow editor. `Pull Request 3181`_ @@ -187,7 +187,7 @@ Enhancements * i18n improvements and French localization. `Pull Request 3762`_ * Should exit immediately to avoid index error - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 3770`_ * Made collection operations help sections slightly more readable. `Pull Request 3784`_ @@ -258,7 +258,7 @@ Enhancements * Change 'User' menu to 'Login or Register' for anonymous users. `Pull Request 3955`_ * Stats docs - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 3968`_ * Enhance filenames with collection context when downloading datasets from GUI. @@ -270,7 +270,7 @@ Enhancements * Update Jupyter image to 17.05 . `Pull Request 4007`_ * Inline CSS in error report email - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 3445`_ * Improve routing for user preference views. `Pull Request 3446`_ @@ -340,7 +340,7 @@ Fixes * Fix margin style values for color picker element. `Pull Request 3965`_ * 17.05 rstudio fixes / one small GIE fix - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 3975`_ * Fix transiently failing Selenium upload test. `Pull Request 3587`_ @@ -480,7 +480,7 @@ Fixes * Fix multiple collection reductions on multiple data parameters. `Pull Request 3875`_ * Fix inability for viz to raise loading with non-ascii messages - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 3881`_ * Fix new history endpoint providing custom builds metadata. `Pull Request 3887`_ @@ -525,7 +525,7 @@ Fixes * Use the configured tool_path when loading data managers. `Pull Request 3989`_ * Typo in remote user / user preferences exception - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4015`_ * Fix setting up explicit container resolution from a container resolution config file. @@ -546,7 +546,7 @@ Fixes * Fix for S3ObjectStore _clean_cache method. `Pull Request 4055`_ * Correct webhooks to use Galaxy.root and be compatible with proxy_prefix - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4065`_ * Fix global grid action event handler. `Pull Request 4068`_ diff --git a/doc/source/releases/17.09.rst b/doc/source/releases/17.09.rst index da30a7417c6..d76a3e701d1 100644 --- a/doc/source/releases/17.09.rst +++ b/doc/source/releases/17.09.rst @@ -80,7 +80,7 @@ Enhancements `Pull Request 4466`_ * Refactor tool error reporting into plugins and implement a new plugin for `influxdb `__ - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4305`_, `Pull Request 4533`_ * Implement second toolshed installation UX targetting the API. `Pull Request 3626`_ @@ -120,10 +120,10 @@ Enhancements (thanks to `@arbernard `__). `Pull Request 4489`_ * Nicer tool click targets for toolbox - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4470`_ * Implement a "plain" bootstrap theme for the UX - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4470`_ * Add Selenium tests for the "saved histories", "published histories", and "custom builds" @@ -136,7 +136,7 @@ Enhancements `Pull Request 4576`_, `Pull Request 4561`_, `Pull Request 4531`_, `Pull Request 4564`_, `Pull Request 4582`_, `Pull Request 4586`_, `Pull Request 4592`_, `Pull Request 4042`_ * Prevent graphite from dying loudly - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4036`_ * Updated parameters check for OpenID (thanks to `@VJalili `__). @@ -164,12 +164,12 @@ Enhancements (thanks to `@bernt-matthias `__). `Pull Request 4096`_ * Security policy - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4113`_ * Improved styling of workflow list. `Pull Request 4138`_ * Swap over all URLs to new wiki location - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4141`_ * Documentation fixes and improvements (thanks to `@nsoranzo `__). @@ -238,10 +238,10 @@ Enhancements (thanks to `@mvdbeek `__). `Pull Request 4271`_ * More uniform backend code for generating page titles - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4272`_, `Pull Request 4578`_ * Fix blockquote>p sizing - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4273`_ * Expose more user properties to admins via the API. `Pull Request 4274`_ @@ -256,7 +256,7 @@ Enhancements (thanks to `@bgruening `__). `Pull Request 4284`_ * Improve security via local network access restrictions - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4289`_ * Enhancement for Phyloviz viz plugin (thanks to `@bgruening `__). @@ -288,7 +288,7 @@ Enhancements (thanks to `@mvdbeek `__). `Pull Request 4368`_, `Pull Request 4380`_ * Galactic Radio Telescope Update - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4376`_ * Show minimum required galaxy version in tool shed and galaxy installation process @@ -308,7 +308,7 @@ Enhancements * Generalize allow_library_path_paste to allow_path_paste. `Pull Request 4404`_ * Small GRT bugfixes - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4408`_ * Update conda channel order to sync with Bioconda (thanks to `@nsoranzo `__). @@ -335,7 +335,7 @@ Enhancements (thanks to `@nsoranzo `__). `Pull Request 4456`_ * Improvements to `Sentry `__ integration - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4457`_, `Pull Request 4471`_ * Fix for GIEs to not automatically uppercase environment variables (thanks to `@xgaia `__). @@ -359,7 +359,7 @@ Enhancements * Minor copy datasets dialog improvement. `Pull Request 4486`_ * 41% reduction in freiburg galaxy startup time - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4495`_ * Allow paramiko cli to recover from ssh timeout (thanks to `@mvdbeek `__). @@ -473,7 +473,7 @@ Enhancements (thanks to `@mvdbeek `__). `Pull Request 4598`_ * Remove various unreachable statements - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4603`_ * Refactored a dataset variable to HDA (thanks to `@VJalili `__). @@ -583,7 +583,7 @@ Fixes (thanks to `@mvdbeek `__). `Pull Request 4420`_ * Some unicoding for local runner - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4426`_ * Break connection in workflow editor if necessary (thanks to `@mvdbeek `__). @@ -603,7 +603,7 @@ Fixes * Fix drag and drop from history for Firefox `Pull Request 4496`_ * 2017-08 Security Patch - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4501`_ * Fix history import if using symlink in database directory (thanks to `@FredericBGA `__). diff --git a/doc/source/releases/17.09_announce.rst b/doc/source/releases/17.09_announce.rst index cb9e14e9b29..708858fa4c8 100644 --- a/doc/source/releases/17.09_announce.rst +++ b/doc/source/releases/17.09_announce.rst @@ -141,7 +141,7 @@ Cross site scripting and session fixation Disclosed on the `mailing list `__ in August 2017. -Vulnerabilities were found by Eric Rasche and Manabu Ishii respectively. Detailed descriptions of these categories of vulnerabilities can be found at: +Vulnerabilities were found by Helena Rasche and Manabu Ishii respectively. Detailed descriptions of these categories of vulnerabilities can be found at: - ``__ - https://www.owasp.org/index.php/Session_fixation diff --git a/doc/source/releases/18.01.rst b/doc/source/releases/18.01.rst index f9bd8a0145e..549774b4547 100644 --- a/doc/source/releases/18.01.rst +++ b/doc/source/releases/18.01.rst @@ -45,7 +45,7 @@ Enhancements (thanks to `@xgaia `__). `Pull Request 4737`_ * Implement a Github error report backend for - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4765`_ * Add maintenance script to simplify setting up a cleanup cron job (thanks to `@scholtalbers `__). @@ -136,7 +136,7 @@ Enhancements * Allow multiple simultaneous uploads via single POST. `Pull Request 4563`_ * Various bits of proactive security-related hardening - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4604`_ * Move test helper ``wait_for_dataset_collection`` for reuse (thanks to `@mvdbeek `__). @@ -157,14 +157,14 @@ Enhancements (thanks to `@nsoranzo `__). `Pull Request 4688`_ * Nicer logging message for sentry - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4693`_ * Use enumeration to provide history input index names. `Pull Request 4700`_ * Cleanup delayed action in ``history-view.js``. `Pull Request 4704`_ * Add logging during for script integrity exceptions - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4720`_ * Fix to respect ``galaxy.json`` supplied metadata for empty outputs. `Pull Request 4722`_ @@ -187,7 +187,7 @@ Enhancements (thanks to `@pcm32 `__ and the `@phnmnl `__ project). `Pull Request 4751`_ * Send all influxdb stats to a single bucket - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 4758`_ * PEP-8 fixes in random doctests. `Pull Request 4764`_ @@ -274,7 +274,7 @@ Enhancements * Document fastest way to iterate on client & client tests together. `Pull Request 5051`_ * Support separate DSN for job sentry error reports - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 5056`_ * Build endpoint separation `Pull Request 5058`_ @@ -531,7 +531,7 @@ Fixes `Pull Request 5045`_ * Safely handle possible ``None`` value encountered in processing and execution of post-job action arguments - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 5050`_ * Restore admin form routes. `Pull Request 5065`_ @@ -563,7 +563,7 @@ Fixes (thanks to `@mvdbeek `__). `Pull Request 5121`_ * Fix attribute error that was missed during refactor of sentry - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 5122`_ * Fix connecting non-input modules to subworkflow inputs. `Pull Request 5140`_ @@ -664,7 +664,7 @@ Fixes (thanks to `@mvdbeek `__). `Pull Request 5357`_ * Handle ``None`` in job owner in API - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 5358`_ * Fix form logic in deactivate or uninstall. `Pull Request 5363`_ diff --git a/doc/source/releases/18.05.rst b/doc/source/releases/18.05.rst index 581b2a8cfe5..eab60edeb07 100644 --- a/doc/source/releases/18.05.rst +++ b/doc/source/releases/18.05.rst @@ -50,7 +50,7 @@ Enhancements `Pull Request 5479`_ * Allow formatting `StatsD `__ data for `InfluxDB `__ - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 5742`_ * Tool versions should follow `PEP 440 `__ (thanks to `@nsoranzo `__). @@ -145,10 +145,10 @@ Enhancements * Various improvements for API tests. `Pull Request 5697`_, `Pull Request 5699`_ * Small log level improvement - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 5715`_ * Swap password checking for the constant time comparison algorithm - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 5719`_ * Upgrade clone failure logging to error (thanks to `@pvanheus `__). @@ -201,7 +201,7 @@ Enhancements (thanks to `@mvdbeek `__). `Pull Request 5924`_ * Document Github bug reporter - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 5928`_ * Rewrite history structure display in `VueJS `__ (thanks to `@anuprulez `__). @@ -295,7 +295,7 @@ Fixes (thanks to `@nsoranzo `__). `Pull Request 5718`_ * `Sentry `__ bug reports working for anonymous users - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 5721`_ * Add Galaxy `lib` directory to Python path for 2 scripts (thanks to `@nsoranzo `__). @@ -366,7 +366,7 @@ Fixes (thanks to `@mvdbeek `__). `Pull Request 5917`_ * Correct signal number in admin documentation - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 5921`_ * Fix error with data managers when trying to move_merge nonexistent extra_files dir diff --git a/doc/source/releases/18.05_announce.rst b/doc/source/releases/18.05_announce.rst index 551738931a5..571b6192d35 100644 --- a/doc/source/releases/18.05_announce.rst +++ b/doc/source/releases/18.05_announce.rst @@ -45,7 +45,7 @@ Highlights **GDPR Beta Mode** We introduce beta GDPR privacy mode. Admins can enable it to help them comply with the European Union law known as `General Data Protection Regulation `__, - which goes into effect on May 25, 2018. Thanks to `@erasche `__. + which goes into effect on May 25, 2018. Thanks to `@hexylena `__. `Pull Request 6069`_ diff --git a/doc/source/releases/18.09.rst b/doc/source/releases/18.09.rst index f133cae3d1a..d39ac530a90 100644 --- a/doc/source/releases/18.09.rst +++ b/doc/source/releases/18.09.rst @@ -54,7 +54,7 @@ Enhancements (thanks to `@nsoranzo `__). `Pull Request 6013`_ * Add hostname metric - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 6021`_ * Pause jobs with invalid inputs (thanks to `@mvdbeek `__). @@ -88,7 +88,7 @@ Enhancements * Common dataset selector `Pull Request 6195`_ * Permit numeric values of GRT instance ID in config - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 6200`_ * Add the newick extension for the Newick datatype, (thanks to `@gregvonkuster `__). @@ -100,7 +100,7 @@ Enhancements * Charts filename download fix `Pull Request 6216`_ * Wrap inputs in labels - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 6221`_ * Workflow submit text links `Pull Request 6240`_ @@ -209,14 +209,14 @@ Enhancements (thanks to `@nsoranzo `__). `Pull Request 6469`_ * Fix bug noticed in sentry - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 6474`_ * Rulebuilder: Special case auto URL schema detection for SRA inputs. `Pull Request 6476`_ * GRT: fix the upload logging init `Pull Request 6478`_ * Add more information to the dataset template - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 6480`_ * Use bootstrap vue classes for workflow import form `Pull Request 6489`_ @@ -344,7 +344,7 @@ Enhancements .. small_enhancement * Remove duplicate metric sending - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 6011`_ * This strips the graphite-specific middleware `Pull Request 6018`_ @@ -670,7 +670,7 @@ Fixes * Redirect unidentified routes in admin panel `Pull Request 6551`_ * Remove the sharing of toolbox in GRT - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 6553`_ * Restore ``log_destination = stdout`` configuration option. `Pull Request 6555`_ diff --git a/doc/source/releases/19.01.rst b/doc/source/releases/19.01.rst index e824f35d890..952205f9e8d 100644 --- a/doc/source/releases/19.01.rst +++ b/doc/source/releases/19.01.rst @@ -310,7 +310,7 @@ Fixes .. bug * Fixes to allow config schema to validate - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 5919`_ * Patch `update.sh` to address dependency update issues on a macOS. `Pull Request 6694`_ @@ -414,7 +414,7 @@ Fixes * Fix resubmission on tool error detection. `Pull Request 7040`_ * GRT: Rip out all parameters, round file sizes - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 7041`_ * Pass through ``--daemon`` option to gunicorn (thanks to `@nsoranzo `__). diff --git a/doc/source/releases/19.01_announce.rst b/doc/source/releases/19.01_announce.rst index d8df49a5cf0..92bc2391b5c 100644 --- a/doc/source/releases/19.01_announce.rst +++ b/doc/source/releases/19.01_announce.rst @@ -21,7 +21,7 @@ Highlights **More Colorful Tags** Use consistently hashed randomized colors for history dataset name tags to increase visual distinction between tags, making it easier to track data flow in complex histories. - Thanks to `@erasche `__. + Thanks to `@hexylena `__. `Pull Request 7072`_, `Pull Request 7081`_ **Extensive Workflow Enhancements (continued)** diff --git a/doc/source/releases/19.05.rst b/doc/source/releases/19.05.rst index 160ab07a718..97592570e4a 100644 --- a/doc/source/releases/19.05.rst +++ b/doc/source/releases/19.05.rst @@ -46,7 +46,7 @@ Enhancements `Pull Request 7193`_, `Pull Request 7192`_, `Pull Request 7537`_, `Pull Request 7540`_, `Pull Request 7684`_, `Pull Request 7704`_ * User facing release notes - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 7527`_ * Implement dataset source and hash tracking. `Pull Request 7487`_, `Pull Request 4659`_, `Pull Request 7549`_ @@ -223,10 +223,10 @@ Enhancements * Automatically set ``cookie_path`` using ``url_for()``. `Pull Request 7404`_ * Allow configuration location of user preferences extra configuration - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 7428`_ * Add hover styling to target tool in tool panel - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 7437`_ * Update Docker testing image (thanks to `@mvdbeek `__). @@ -260,7 +260,7 @@ Enhancements * Refactor XSD output elements to share common output attributes. `Pull Request 7555`_ * Improve linking between job conf handler assignment and documentation - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 7558`_ * Refactor ``galaxy.web.security`` into ``galaxy.security.idencoding``. `Pull Request 7560`_ @@ -537,7 +537,7 @@ Fixes (thanks to `@mvdbeek `__). `Pull Request 7547`_ * Correct db-self handler assignment method - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 7561`_ * Fix database config doctests when env vars are in use for dburi `Pull Request 7562`_ @@ -549,7 +549,7 @@ Fixes * Update cloud imports to adhere with cloudbridge interface changes. `Pull Request 7586`_ * Fix RStudio GIE - (thanks to `@erasche `__). + (thanks to `@hexylena `__). `Pull Request 7590`_, `Pull Request 7925`_ * Fix styleguide. `Pull Request 7602`_ diff --git a/doc/source/releases/19.05_announce.rst b/doc/source/releases/19.05_announce.rst index 557133feea8..7716266dfc8 100644 --- a/doc/source/releases/19.05_announce.rst +++ b/doc/source/releases/19.05_announce.rst @@ -14,7 +14,7 @@ Highlights oriented but will likely be even more so now. Features such as favoriting tools, improved workflow editor feedback, and an exciting new cellxgene GIE may have been featured in these release notes in the past but are now featured in the - user release notes instead. A huge thanks to `@erasche `__ + user release notes instead. A huge thanks to `@hexylena `__ for getting the ball rolling on this initiative. Check out the `19.05 user release notes `__. diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index e97c27833a7..1876bdb82bd 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -104,6 +104,8 @@ class AuthnzManager(object): self.oidc_backends_config[idp] = self._parse_custos_config(child) self.oidc_backends_implementation[idp] = 'custos' self.app.config.oidc.append(idp) + else: + raise ParseError("Unknown provider specified") if len(self.oidc_backends_config) == 0: raise ParseError("No valid provider configuration parsed.") except ImportError: diff --git a/lib/galaxy/config/config_manage.py b/lib/galaxy/config/config_manage.py index aacfea0c046..f88eff40452 100644 --- a/lib/galaxy/config/config_manage.py +++ b/lib/galaxy/config/config_manage.py @@ -101,6 +101,12 @@ UWSGI_OPTIONS = OrderedDict([ 'default': '/favicon.ico=static/favicon.ico', 'type': 'str', }), + ('static-safe', { + 'key': 'static-safe', + 'desc': """Allow serving images out of `client`. Most modern Galaxy interfaces bundle all of this, but some older pages still serve these via symlink, requiring this rule.""", + 'default': 'client/galaxy/images', + 'type': 'str', + }), ('master', { 'desc': """Enable the master process manager. Disabled by default for maximum compatibility with CTRL+C, but should be enabled for use with --daemon and/or production deployments.""", 'default': False, diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index 05a7666e407..9fa6e76623f 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -50,6 +50,11 @@ uwsgi: # Mapping to serve the favicon. static-map: /favicon.ico=static/favicon.ico + # Allow serving images out of `client`. Most modern Galaxy interfaces + # bundle all of this, but some older pages still serve these via + # symlink, requiring this rule. + static-safe: client/galaxy/images + # Enable the master process manager. Disabled by default for maximum # compatibility with CTRL+C, but should be enabled for use with # --daemon and/or production deployments. @@ -1785,6 +1790,49 @@ galaxy: # environment individually. #environment_setup_file: null + # Enable export of Galaxy Markdown documents (pages and workflow + # reports) to PDF. Requires manual installation and setup of + # weasyprint (latest version available for Python 2.7 is 0.42). + #enable_beta_markdown_export: false + + # CSS file to apply to all Markdown exports to PDF - currently used by + # WeasyPrint during rendering an HTML export of the document to PDF. + #markdown_export_css: markdown_export.css + + # CSS file to apply to "Galaxy Page" exports to PDF. Generally prefer + # markdown_export_css, but this is here for deployments that would + # like to tailor different kinds of exports. + #markdown_export_css_pages: markdown_export_pages.css + + # CSS file to apply to invocation report exports to PDF. Generally + # prefer markdown_export_css, but this is here for deployments that + # would like to tailor different kinds of exports. + #markdown_export_css_invocation_reports: markdown_export_invocation_reports.css + + # Prologue Markdown/HTML to apply to markdown exports to PDF. Allowing + # branded headers. + #markdown_export_prologue: '' + + # Prologue Markdown/HTML to apply to markdown exports to PDF. Allowing + # branded footers. + #markdown_export_epilogue: '' + + # Alternative to markdown_export_prologue that applies just to page + # exports. + #markdown_export_prologue_pages: '' + + # Alternative to markdown_export_prologue that applies just to + # invocation report exports. + #markdown_export_prologue_invocation_reports: '' + + # Alternative to markdown_export_epilogue that applies just to page + # exports. + #markdown_export_epilogue_pages: '' + + # Alternative to markdown_export_epilogue that applies just to + # invocation report exports. + #markdown_export_epilogue_invocation_reports: '' + # Optional file containing job resource data entry fields definition. # These fields will be presented to users in the tool forms and allow # them to overwrite default job resources such as number of diff --git a/lib/galaxy/config/sample/reports.yml.sample b/lib/galaxy/config/sample/reports.yml.sample index a0d21cbc3a0..b207fd53bd7 100644 --- a/lib/galaxy/config/sample/reports.yml.sample +++ b/lib/galaxy/config/sample/reports.yml.sample @@ -32,6 +32,11 @@ uwsgi: # Mapping to serve the favicon. static-map: /favicon.ico=static/favicon.ico + # Allow serving images out of `client`. Most modern Galaxy interfaces + # bundle all of this, but some older pages still serve these via + # symlink, requiring this rule. + static-safe: client/galaxy/images + # Enable the master process manager. Disabled by default for maximum # compatibility with CTRL+C, but should be enabled for use with # --daemon and/or production deployments. diff --git a/lib/galaxy/config/sample/tool_shed.yml.sample b/lib/galaxy/config/sample/tool_shed.yml.sample index b8e5b169221..236e4b7430c 100644 --- a/lib/galaxy/config/sample/tool_shed.yml.sample +++ b/lib/galaxy/config/sample/tool_shed.yml.sample @@ -32,6 +32,11 @@ uwsgi: # Mapping to serve the favicon. static-map: /favicon.ico=static/favicon.ico + # Allow serving images out of `client`. Most modern Galaxy interfaces + # bundle all of this, but some older pages still serve these via + # symlink, requiring this rule. + static-safe: client/galaxy/images + # Enable the master process manager. Disabled by default for maximum # compatibility with CTRL+C, but should be enabled for use with # --daemon and/or production deployments. diff --git a/lib/galaxy/config/script.py b/lib/galaxy/config/script.py index f8bcb54cd66..03cfc0cd22f 100644 --- a/lib/galaxy/config/script.py +++ b/lib/galaxy/config/script.py @@ -37,6 +37,7 @@ DEFAULT_DB_CONN = 'sqlite:///./database/universe.sqlite?isolation_level=IMMEDIAT SAMPLES_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), 'sample')) GALAXY_CONFIG_TEMPLATE_FILE = os.path.join(SAMPLES_PATH, 'galaxy.yml.sample') STATIC_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'web', 'framework', 'static')) +CLIENT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 'client')) MSG_CONFIG_SUMMARY = """ For help on configuring Galaxy, consult the documentation at: \n {} @@ -55,6 +56,7 @@ GALAXY_CONFIG_SUBSTITUTIONS = { ' static-map: /static/style=static/style/blue': ' static-map: /static=${static_path}/style/blue', ' static-map: /static=static': ' static-map: /static=${static_path}', ' static-map: /favicon.ico=static/favicon.ico': ' static-map: /static=${static_path}/favicon.ico', + ' static-safe: client/galaxy/images': ' ${client_path}/galaxy/images', ' virtualenv: .venv': ' #venv: .venv # not used when running installed', ' pythonpath: lib': ' #pythonpath: lib # not used when running installed', ' #config_dir: false': ' config_dir: ${config_dir}', @@ -147,6 +149,7 @@ def _handle_galaxy_yml(args, config_dir, data_dir): uwsgi_transport=uwsgi_transport, config_dir=config_dir, data_dir=data_dir, + client_dir=CLIENT_PATH, static_path=STATIC_PATH, database_connection=args.db_conn, ) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index a8d5d73d7b1..e88bd656a2a 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -1119,8 +1119,8 @@ class Cool(H5): if super(Cool, self).sniff(filename): keys = ['chroms', 'bins', 'pixels', 'indexes'] with h5py.File(filename, 'r') as handle: - fmt = handle.attrs.get('format') - url = handle.attrs.get('format-url') + fmt = util.unicodify(handle.attrs.get('format')) + url = util.unicodify(handle.attrs.get('format-url')) if fmt == MAGIC or url == URL: if not all(name in handle.keys() for name in keys): return False @@ -1179,8 +1179,8 @@ class MCool(H5): return False res0 = list(handle['resolutions'].keys())[0] keys = ['chroms', 'bins', 'pixels', 'indexes'] - fmt = handle['resolutions'][res0].attrs.get('format') - url = handle['resolutions'][res0].attrs.get('format-url') + fmt = util.unicodify(handle['resolutions'][res0].attrs.get('format')) + url = util.unicodify(handle['resolutions'][res0].attrs.get('format-url')) if fmt == MAGIC or url == URL: if not all(name in handle['resolutions'][res0].keys() for name in keys): return False diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 29691a9c4a6..4d398f0f230 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -11,7 +11,10 @@ import pkg_resources import yaml from galaxy.containers import parse_containers_config -from galaxy.util import asbool +from galaxy.util import ( + asbool, + which, +) from galaxy.util.properties import ( find_config_file, load_app_properties @@ -145,7 +148,7 @@ class ConditionalDependencies(object): return "galaxy.jobs.runners.pbs:PBSJobRunner" in self.job_runners def check_pykube(self): - return "galaxy.jobs.runners.kubernetes:KubernetesJobRunner" in self.job_runners + return "galaxy.jobs.runners.kubernetes:KubernetesJobRunner" in self.job_runners or which('kubectl') def check_chronos_python(self): return "galaxy.jobs.runners.chronos:ChronosJobRunner" in self.job_runners @@ -194,7 +197,7 @@ class ConditionalDependencies(object): def optional(config_file=None): if not config_file: - config_file = find_config_file(['galaxy', 'universe_wsgi']) + config_file = find_config_file(['galaxy', 'universe_wsgi'], include_samples=True) if not config_file: print("galaxy.dependencies.optional: no config file found", file=sys.stderr) return [] diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt index 02c01a9bc06..34360b2b315 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt @@ -4,47 +4,47 @@ alabaster==0.7.12 argh==0.26.2 atomicwrites==1.3.0 attrs==19.3.0 -babel==2.7.0 +babel==2.8.0 certifi==2019.11.28 chardet==3.0.4 commonmark==0.9.1 configparser==4.0.2 ; python_version < '3.2' contextlib2==0.6.0.post1 ; python_version < '3.5' -coverage==4.5.4 +coverage==5.0.2 docutils==0.15.2 funcsigs==1.0.2 ; python_version < '3.3' -future==0.18.2 +future==0.18.2 ; python_version < '3' gunicorn==19.10.0 idna==2.8 -imagesize==1.1.0 -importlib-metadata==1.2.0 ; python_version < '3.8' +imagesize==1.2.0 +importlib-metadata==1.3.0 ; python_version < '3.8' jinja2==2.10.3 lxml==4.4.2 -markdown==3.0.1 +markdown==3.1.1 markupsafe==1.1.1 mirakuru==1.1.0 mock==3.0.5 more-itertools==5.0.0 nose==1.3.7 nosehtml==0.4.5 -packaging==19.2 +packaging==20.0 pathlib2==2.3.2 ; python_version < '3.6' pathtools==0.1.2 pluggy==0.13.1 port-for==0.4 psutil==5.6.7 -py==1.8.0 +py==1.8.1 pygithub3==0.5.1 ; python_version < '3' pygments==2.5.2 -pyparsing==2.4.5 +pyparsing==2.4.6 pytest-cov==2.8.1 pytest-html==1.22.1 pytest-metadata==1.8.0 pytest-postgresql==1.4.1 pytest-pythonpath==0.7.3 -pytest==4.6.6 +pytest==4.6.9 pytz==2019.3 -pyyaml==5.2 +pyyaml==5.3 recommonmark==0.6.0 requests==2.22.0 scandir==1.10.0 ; python_version < '3.5' @@ -60,5 +60,5 @@ twill==0.9.1 ; python_version < '3' typing==3.7.4.1 ; python_version < '3.5' urllib3==1.25.7 ; python_version == '2.7' watchdog==0.9.0 -wcwidth==0.1.7 ; sys_platform != 'win32' +wcwidth==0.1.8 ; sys_platform != 'win32' zipp==0.6.0 diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 4214b7cec27..c0463e70b71 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -18,7 +18,7 @@ azure-nspkg==3.0.2 azure-storage-blob==1.3.1 azure-storage-common==1.4.2 azure-storage-nspkg==3.1.0 -babel==2.7.0 +babel==2.8.0 bagit==1.7.0 bcrypt==3.1.7 bdbag==1.5.6 @@ -37,7 +37,7 @@ certifi==2019.11.28 cffi==1.13.2 chardet==3.0.4 cheetah3==3.2.4 -cliff==2.16.0 +cliff==2.17.0 cloudauthz==0.6.0 cloudbridge==2.0.0 cmd2==0.8.9 @@ -53,22 +53,21 @@ dictobj==0.4 docopt==0.6.2 docutils==0.15.2 dogpile.cache==0.9.0 -ecdsa==0.14.1 +ecdsa==0.15 enum34==1.1.6 ; python_version < '3.4' fabric3==1.14.post1 funcsigs==1.0.2 ; python_version < '3.3' functools32==3.2.3.post2 ; python_version < '3.2' -future==0.18.2 futures==3.3.0 ; python_version == '2.6' or python_version == '2.7' galaxy-sequence-utils==1.1.3 google-api-python-client==1.7.8 google-auth-httplib2==0.0.3 -google-auth==1.7.1 +google-auth==1.10.0 gxformat2==0.10.1 h5py==2.10.0 -httplib2==0.14.0 +httplib2==0.15.0 idna==2.8 -importlib-metadata==1.2.0 ; python_version < '3.8' +importlib-metadata==1.3.0 ; python_version < '3.8' ipaddress==1.0.23 ; python_version < '3.3' isa-rwval==0.10.7 iso8601==0.1.12 @@ -78,12 +77,13 @@ jsonpatch==1.24 jsonpointer==2.0 jsonschema==3.2.0 keystoneauth1==3.18.0 -kombu==4.6.6 +kombu==4.6.7 lockfile==0.12.2 lxml==4.4.2 mako==1.1.0 +markdown==3.1.1 markupsafe==1.1.1 -mercurial==5.2 +mercurial==5.2.2 mistune==0.8.4 monotonic==1.5 more-itertools==5.0.0 @@ -95,25 +95,25 @@ mypy-extensions==0.4.3 netaddr==0.7.19 netifaces==0.10.9 networkx==1.11 -nodeenv==1.3.3 +nodeenv==1.3.4 nose==1.3.7 -numpy==1.16.5 +numpy==1.16.6 oauth2client==4.1.3 oauthlib==3.1.0 openstacksdk==0.17.0 os-client-config==1.33.0 os-service-types==1.7.0 -osc-lib==1.14.1 -oslo.config==6.12.0 +osc-lib==1.15.0 +oslo.config==7.0.0 oslo.context==2.23.0 -oslo.i18n==3.25.0 -oslo.log==3.45.0 +oslo.i18n==3.25.1 +oslo.log==3.45.2 oslo.serialization==2.29.2 -oslo.utils==3.42.0 -packaging==19.2 -paramiko==2.7.0 +oslo.utils==3.42.1 +packaging==20.0 +paramiko==2.7.1 parsley==1.3 -paste==3.2.3 +paste==3.2.4 pastedeploy==2.0.1 pastescript==3.2.0 pathlib2==2.3.2 ; python_version < '3.6' @@ -132,16 +132,16 @@ pyjwt==1.7.1 pykwalify==1.7.0 pynacl==1.3.0 pyopenssl==19.1.0 -pyparsing==2.4.5 +pyparsing==2.4.6 pyperclip==1.7.0 -pyrsistent==0.15.6 +pyrsistent==0.15.7 pysam==0.15.2 pysftp==0.2.9 python-cinderclient==4.0.0 python-dateutil==2.8.1 python-genomespaceclient==1.2.0 python-glanceclient==2.12.0 -python-jose==3.0.1 +python-jose==3.1.0 python-keystoneclient==3.17.0 python-neutronclient==6.9.0 python-novaclient==11.0.0 @@ -149,7 +149,7 @@ python-openid==2.2.5 ; python_version < '3.0' python-swiftclient==3.6.0 pytz==2019.3 pyuwsgi==2.0.18.post0 -pyyaml==5.2 +pyyaml==5.3 rdflib-jsonld==0.4.0 rdflib==4.2.2 repoze.lru==0.7 @@ -171,8 +171,8 @@ simplejson==3.17.0 six==1.11.0 social-auth-core[openidconnect]==3.1.0+gx0 sqlalchemy-migrate==0.13.0 -sqlalchemy-utils==0.35.0 -sqlalchemy==1.3.11 +sqlalchemy-utils==0.36.1 +sqlalchemy==1.3.12 sqlparse==0.3.0 stevedore==1.31.0 subprocess32==3.5.4 ; python_version < '3.0' @@ -183,11 +183,11 @@ typing-extensions==3.7.4.1 typing==3.7.4.1 ; python_version < '3.5' tzlocal==2.0.0 unicodecsv==0.14.1 ; python_version < '3.0' -uritemplate==3.0.0 +uritemplate==3.0.1 urllib3==1.25.7 ; python_version == '2.7' vine==1.3.0 warlock==1.3.3 -wcwidth==0.1.7 ; sys_platform != 'win32' +wcwidth==0.1.8 ; sys_platform != 'win32' webencodings==0.5.1 webob==1.8.5 whoosh==2.7.4 diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index ef942469d48..8b9d67523ca 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1034,11 +1034,11 @@ class JobWrapper(HasResourceParameters): return False def get_version_string_path(self): - return os.path.abspath(os.path.join(self.working_directory, COMMAND_VERSION_FILENAME)) + return os.path.abspath(os.path.join(self.working_directory, "outputs", COMMAND_VERSION_FILENAME)) - # TODO: Remove in Galaxy 20.XX, for running jobs at GX upgrade + # TODO: Remove in Galaxy 21.XX, for running jobs at GX upgrade def get_version_string_path_legacy(self): - return os.path.abspath(os.path.join(self.app.config.new_file_path, "GALAXY_VERSION_STRING_%s" % self.job_id)) + return os.path.abspath(os.path.join(self.working_directory, COMMAND_VERSION_FILENAME)) def __prepare_upload_paramfile(self, tool_evaluator): """Special case paramfile handling for the upload tool. Moves the paramfile to the working directory @@ -2111,7 +2111,7 @@ class JobWrapper(HasResourceParameters): "connection_configuration": container.connection_configuration, }, f) - return "(python '%s'/lib/galaxy_ext/container_monitor/monitor.py &) " % exec_dir + return "(python '%s'/lib/galaxy_ext/container_monitor/monitor.py &); sleep 1 " % exec_dir @property def user(self): diff --git a/lib/galaxy/jobs/mapper.py b/lib/galaxy/jobs/mapper.py index 8d50ad0a9b8..bf07063b1e5 100644 --- a/lib/galaxy/jobs/mapper.py +++ b/lib/galaxy/jobs/mapper.py @@ -15,6 +15,11 @@ DYNAMIC_DESTINATION_ID = "dynamic_legacy_from_url" ERROR_MESSAGE_NO_RULE_FUNCTION = "Galaxy misconfigured - cannot find dynamic rule function name for destination %s." ERROR_MESSAGE_RULE_FUNCTION_NOT_FOUND = "Galaxy misconfigured - no rule function named %s found in dynamic rule modules." +ERROR_MESSAGE_RULE_EXCEPTION = "Encountered an unhandled exception while caching job destination dynamic rule." + + +class JobMappingConfigurationException(Exception): + pass class JobMappingException(Exception): @@ -157,12 +162,12 @@ class JobRunnerMapper(object): rule_modules, expand_function_name) if not expand_function: message = ERROR_MESSAGE_RULE_FUNCTION_NOT_FOUND % expand_function_name - raise Exception(message) + raise JobMappingConfigurationException(message) else: expand_function = self.__find_function_by_tool_id(rule_modules) if not expand_function: message = ERROR_MESSAGE_NO_RULE_FUNCTION % destination - raise Exception(message) + raise JobMappingConfigurationException(message) return expand_function def __get_rule_modules_or_defaults(self, rules_module_name): @@ -192,7 +197,7 @@ class JobRunnerMapper(object): elif expand_type in STOCK_RULES: expand_function = STOCK_RULES[expand_type] else: - raise Exception("Unhandled dynamic job runner type specified - %s" % expand_type) + raise JobMappingConfigurationException("Unhandled dynamic job runner type specified - %s" % expand_type) return self.__handle_rule(expand_function, destination) @@ -225,8 +230,15 @@ class JobRunnerMapper(object): return job_destination def __cache_job_destination(self, params, raw_job_destination=None): - self.cached_job_destination = self.__determine_job_destination( - params, raw_job_destination=raw_job_destination) + try: + self.cached_job_destination = self.__determine_job_destination(params, raw_job_destination=raw_job_destination) + except (JobMappingConfigurationException, JobMappingException, JobNotReadyException): + raise + except Exception: + # Other exceptions should not bubble up to the job wrapper since they can occur during the fail() method, + # causing jobs to become permanently stuck in a non-terminal state. + log.exception("Caught unhandled exception while attempting to cache job destination:") + raise JobMappingException(ERROR_MESSAGE_RULE_EXCEPTION) return self.cached_job_destination def get_job_destination(self, params): diff --git a/lib/galaxy/jobs/runners/cli.py b/lib/galaxy/jobs/runners/cli.py index 2da5643a6ce..6508142674b 100644 --- a/lib/galaxy/jobs/runners/cli.py +++ b/lib/galaxy/jobs/runners/cli.py @@ -161,10 +161,6 @@ class ShellJobRunner(AsynchronousJobRunner): if ajs.job_wrapper.get_state() == model.Job.states.DELETED: continue - external_metadata = not asbool(ajs.job_wrapper.job_destination.params.get("embed_metadata_in_job", DEFAULT_EMBED_METADATA_IN_JOB)) - if external_metadata: - self._handle_metadata_externally(ajs.job_wrapper, resolve_requirements=True) - log.debug("(%s/%s) job not found in batch state check" % (id_tag, external_job_id)) shell_params, job_params = self.parse_destination_params(ajs.job_destination.params) shell, job_interface = self.get_cli_plugins(shell_params, job_params) @@ -187,6 +183,9 @@ class ShellJobRunner(AsynchronousJobRunner): ajs.running = True ajs.old_state = state if state == model.Job.states.OK: + external_metadata = not asbool(ajs.job_wrapper.job_destination.params.get("embed_metadata_in_job", DEFAULT_EMBED_METADATA_IN_JOB)) + if external_metadata: + self.work_queue.put((self.handle_metadata_externally, ajs)) log.debug('(%s/%s) job execution finished, running job wrapper finish method' % (id_tag, external_job_id)) self.work_queue.put((self.finish_job, ajs)) else: @@ -194,6 +193,9 @@ class ShellJobRunner(AsynchronousJobRunner): # Replace the watch list with the updated version self.watched = new_watched + def handle_metadata_externally(self, ajs): + self._handle_metadata_externally(ajs.job_wrapper, resolve_requirements=True) + def __handle_out_of_memory(self, ajs, external_job_id): shell_params, job_params = self.parse_destination_params(ajs.job_destination.params) shell, job_interface = self.get_cli_plugins(shell_params, job_params) diff --git a/lib/galaxy/jobs/runners/util/cli/shell/local.py b/lib/galaxy/jobs/runners/util/cli/shell/local.py index 6132655b5ff..c4deeeb3624 100644 --- a/lib/galaxy/jobs/runners/util/cli/shell/local.py +++ b/lib/galaxy/jobs/runners/util/cli/shell/local.py @@ -34,6 +34,8 @@ class LocalShell(BaseShellExec): True >>> exec_result.stdout.strip() == u'Hello World' True + >>> exec_result.returncode + 0 >>> exec_result = exec_python("import time; time.sleep(10)", timeout=1, timeout_check_interval=.1) >>> exec_result.stdout == u'' True @@ -63,6 +65,8 @@ class LocalShell(BaseShellExec): kill_pg(p.pid) return Bunch(stdout=u'', stderr=TIMEOUT_ERROR_MESSAGE, returncode=TIMEOUT_RETURN_CODE) outf.seek(0) + # Need to poll once to establish return code + p.poll() return Bunch(stdout=_read_str(outf), stderr=_read_str(p.stderr), returncode=p.returncode) diff --git a/lib/galaxy/jobs/runners/util/cli/shell/rsh.py b/lib/galaxy/jobs/runners/util/cli/shell/rsh.py index 2352208229a..c6c7e1fbd1a 100644 --- a/lib/galaxy/jobs/runners/util/cli/shell/rsh.py +++ b/lib/galaxy/jobs/runners/util/cli/shell/rsh.py @@ -6,7 +6,8 @@ from pulsar.managers.util.retry import RetryActionExecutor from galaxy.util import ( smart_str, - unicodify + string_as_bool, + unicodify, ) from galaxy.util.bunch import Bunch from .local import LocalShell @@ -40,11 +41,11 @@ class RemoteShell(LocalShell): class SecureShell(RemoteShell): - SSH_NEW_KEY_STRING = 'Are you sure you want to continue connecting' def __init__(self, rsh='ssh', rcp='scp', private_key=None, port=None, strict_host_key_checking=True, **kwargs): - strict_host_key_checking = "yes" if strict_host_key_checking else "no" - options = ["-o", "StrictHostKeyChecking=%s" % strict_host_key_checking] + options = [] + if not string_as_bool(strict_host_key_checking): + options.extend(["-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null"]) options.extend(["-o", "ConnectTimeout=60"]) if private_key: options.extend(['-i', private_key]) @@ -55,13 +56,14 @@ class SecureShell(RemoteShell): class ParamikoShell(object): - def __init__(self, username, hostname, password=None, private_key=None, port=22, timeout=60, **kwargs): + def __init__(self, username, hostname, password=None, private_key=None, port=22, timeout=60, strict_host_key_checking=True, **kwargs): self.username = username self.hostname = hostname self.password = password self.private_key = private_key self.port = int(port) if port else port self.timeout = int(timeout) if timeout else timeout + self.strict_host_key_checking = string_as_bool(strict_host_key_checking) self.ssh = None self.retry_action_executor = RetryActionExecutor(max_retries=100, interval_max=300) self.connect() @@ -69,7 +71,7 @@ class ParamikoShell(object): def connect(self): log.info("Attempting establishment of new paramiko SSH channel") self.ssh = paramiko.SSHClient() - self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.ssh.set_missing_host_key_policy(paramiko.RejectPolicy() if self.strict_host_key_checking else paramiko.WarningPolicy()) self.ssh.connect(hostname=self.hostname, port=self.port, username=self.username, diff --git a/lib/galaxy/managers/interactivetool.py b/lib/galaxy/managers/interactivetool.py index dd740f96eb1..a6504c5c587 100644 --- a/lib/galaxy/managers/interactivetool.py +++ b/lib/galaxy/managers/interactivetool.py @@ -270,7 +270,7 @@ class InteractiveToolManager(object): entry_point = trans.sa_session.query(model.InteractiveToolEntryPoint).get(entry_point_id) if self.app.interactivetool_manager.can_access_entry_point(trans, entry_point): if entry_point.active: - return self.target_if_active(entry_point) + return self.target_if_active(trans, entry_point) elif entry_point.deleted: raise exceptions.MessageException('InteractiveTool has ended. You will have to start a new one.') else: diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index 58b00721a33..54b119f9033 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -66,6 +66,15 @@ class JobManager(object): trans.sa_session.refresh(job) return job + def stop(self, job, message=None): + if not job.finished: + job.mark_deleted(self.app.config.track_jobs_in_database) + self.app.model.context.current.flush() + self.app.job_manager.stop(job, message=message) + return True + else: + return False + class JobSearch(object): """Search for jobs using tool inputs or other jobs""" diff --git a/lib/galaxy/managers/users.py b/lib/galaxy/managers/users.py index 5d1b5ae9283..71ff455a74d 100644 --- a/lib/galaxy/managers/users.py +++ b/lib/galaxy/managers/users.py @@ -106,12 +106,7 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): # TODO:?? flush needed for permissions below? If not, make optional except exc.IntegrityError as db_err: raise exceptions.Conflict(str(db_err)) - # can throw an sqlalx.IntegrityError if username not unique - self.app.security_agent.create_private_user_role(user) - # We set default user permissions, before we log in and set the default history permissions - if hasattr(self.app.config, "new_user_dataset_access_role_default_private"): - permissions = self.app.config.new_user_dataset_access_role_default_private - self.app.security_agent.user_set_default_permissions(user, default_access_private=permissions) + self.app.security_agent.create_user_role(user, self.app) return user def delete(self, user, flush=True): @@ -522,6 +517,11 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): log.exception('Subscribing to the mailing list has failed.') return "Subscribing to the mailing list has failed." + def activate(self, user): + user.active = True + self.session().add(user) + self.session().flush() + class UserSerializer(base.ModelSerializer, deletable.PurgableSerializerMixin): model_manager_class = UserManager diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index e687b521df0..e77c9de7920 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -659,7 +659,7 @@ class TaskMetricNumeric(BaseJobMetric, RepresentById): class Job(JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): dict_collection_visible_keys = ['id', 'state', 'exit_code', 'update_time', 'create_time', 'galaxy_version'] - dict_element_visible_keys = ['id', 'state', 'exit_code', 'update_time', 'create_time', 'galaxy_version'] + dict_element_visible_keys = ['id', 'state', 'exit_code', 'update_time', 'create_time', 'galaxy_version', 'command_version'] """ A job represents a request to run a tool given input datasets, tool @@ -1132,6 +1132,12 @@ class Job(JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): config_value = default return config_value + @property + def command_version(self): + # TODO: make actual database property and track properly - we should be recording this on the job and not on the datasets + for dataset_assoc in self.output_datasets: + return dataset_assoc.dataset.tool_version + class Task(JobLike, RepresentById): """ diff --git a/lib/galaxy/model/security.py b/lib/galaxy/model/security.py index 698942a7d00..6505d60f521 100644 --- a/lib/galaxy/model/security.py +++ b/lib/galaxy/model/security.py @@ -656,6 +656,18 @@ class GalaxyRBACAgent(RBACAgent): self.sa_session.flush() return assoc + def create_user_role(self, user, app): + # Create private user role if necessary + if not self.get_private_user_role(user): + self.create_private_user_role(user) + # Create default user permissions if necessary + if not user.default_permissions: + if hasattr(app.config, "new_user_dataset_access_role_default_private"): + permissions = app.config.new_user_dataset_access_role_default_private + self.user_set_default_permissions(user, default_access_private=permissions) + else: + self.user_set_default_permissions(user, history=True, dataset=True) + def create_private_user_role(self, user): # Create private role role = self.model.Role(name=user.email, description='Private Role for ' + user.email, type=self.model.Role.types.PRIVATE) diff --git a/lib/galaxy/model/tool_shed_install/__init__.py b/lib/galaxy/model/tool_shed_install/__init__.py index c9364c43485..a48f0509369 100644 --- a/lib/galaxy/model/tool_shed_install/__init__.py +++ b/lib/galaxy/model/tool_shed_install/__init__.py @@ -137,7 +137,7 @@ class ToolShedRepository(object): self.shed_config_filename = shed_tool_conf_dict['config_filename'] return shed_tool_conf_dict # Very last resort, get default shed_tool_config file for this instance - shed_tool_conf_dict = self.app.toolbox.default_shed_tool_conf_dict() + shed_tool_conf_dict = app.toolbox.default_shed_tool_conf_dict() self.shed_config_filename = shed_tool_conf_dict['config_filename'] return shed_tool_conf_dict diff --git a/lib/galaxy/tool_util/deps/container_classes.py b/lib/galaxy/tool_util/deps/container_classes.py index 9970334fdea..aaab9a05650 100644 --- a/lib/galaxy/tool_util/deps/container_classes.py +++ b/lib/galaxy/tool_util/deps/container_classes.py @@ -179,7 +179,7 @@ class HasDockerLikeVolumes(object): if self.job_info.tool_directory: defaults += ",$tool_directory:default_ro" if self.job_info.job_directory: - defaults += ",$job_directory:default_ro" + defaults += ",$job_directory:default_ro,$job_directory/outputs:rw" if self.job_info.tmp_directory is not None: defaults += ",$tmp_directory:rw" if self.job_info.home_directory is not None: diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build.py b/lib/galaxy/tool_util/deps/mulled/mulled_build.py index 412994c3152..6150f312d3e 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build.py @@ -180,7 +180,7 @@ def mull_targets( if DEST_BASE_IMAGE: dest_base_image = DEST_BASE_IMAGE else: - dest_base_image = DEFAULT_EXTENDED_BASE_IMAGE if not any_target_requires_extended_base(targets) else DEST_BASE_IMAGE + dest_base_image = DEFAULT_EXTENDED_BASE_IMAGE if any_target_requires_extended_base(targets) else DEST_BASE_IMAGE targets = list(targets) if involucro_context is None: diff --git a/lib/galaxy/tool_util/deps/mulled/util.py b/lib/galaxy/tool_util/deps/mulled/util.py index d005d89ffd6..57e76127180 100644 --- a/lib/galaxy/tool_util/deps/mulled/util.py +++ b/lib/galaxy/tool_util/deps/mulled/util.py @@ -8,7 +8,6 @@ import re import sys import tarfile import threading -import time from io import BytesIO import packaging.version @@ -293,13 +292,13 @@ def split_container_name(name): class PrintProgress(object): def __init__(self): self.thread = threading.Thread(target=self.progress) - self.stop = False + self.stop = threading.Event() def progress(self): - while not self.stop: + while not self.stop.is_set(): print(".", end="") sys.stdout.flush() - time.sleep(60) + self.stop.wait(60) print("") def __enter__(self): @@ -307,7 +306,7 @@ class PrintProgress(object): return self def __exit__(self, exc_type, exc_val, exc_tb): - self.stop = True + self.stop.set() self.thread.join() diff --git a/lib/galaxy/tool_util/parser/cwl.py b/lib/galaxy/tool_util/parser/cwl.py index f08b48ec0d9..6e729ec9c28 100644 --- a/lib/galaxy/tool_util/parser/cwl.py +++ b/lib/galaxy/tool_util/parser/cwl.py @@ -100,6 +100,9 @@ class CwlToolSource(ToolSource): def parse_description(self): return self.tool_proxy.description() + def parse_interactivetool(self): + return [] + def parse_input_pages(self): page_source = CwlPageSource(self.tool_proxy) return PagesSource([page_source]) diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py index 9aaddfa09ee..f38301762e0 100644 --- a/lib/galaxy/tool_util/parser/xml.py +++ b/lib/galaxy/tool_util/parser/xml.py @@ -531,6 +531,7 @@ def _test_elem_to_dict(test_elem, i): inputs=__parse_input_elems(test_elem, i), expect_num_outputs=test_elem.get("expect_num_outputs"), command=__parse_assert_list_from_elem(test_elem.find("assert_command")), + command_version=__parse_assert_list_from_elem(test_elem.find("assert_command_version")), stdout=__parse_assert_list_from_elem(test_elem.find("assert_stdout")), stderr=__parse_assert_list_from_elem(test_elem.find("assert_stderr")), expect_exit_code=test_elem.get("expect_exit_code"), diff --git a/lib/galaxy/tool_util/provided_metadata.py b/lib/galaxy/tool_util/provided_metadata.py index 6c68d4488b6..b8be575e4c4 100644 --- a/lib/galaxy/tool_util/provided_metadata.py +++ b/lib/galaxy/tool_util/provided_metadata.py @@ -110,7 +110,7 @@ class LegacyToolProvidedMetadata(BaseToolProvidedMetadata): line = stringify_dictionary_keys(json.loads(line)) assert 'type' in line except Exception: - log.exception('(%s) Got JSON data from tool, but data is improperly formatted or no "type" key in data' % job_wrapper.job_id) + log.exception('(%s) Got JSON data from tool, but data is improperly formatted or no "type" key in data' % getattr(job_wrapper, "job_id", None)) log.debug('Offending data was: %s' % line) continue # Set the dataset id if it's a dataset entry and isn't set. diff --git a/lib/galaxy/tool_util/verify/interactor.py b/lib/galaxy/tool_util/verify/interactor.py index fc404959ee0..cac93ddd30d 100644 --- a/lib/galaxy/tool_util/verify/interactor.py +++ b/lib/galaxy/tool_util/verify/interactor.py @@ -901,6 +901,7 @@ def _verify_outputs(testdef, history, jobs, tool_id, data_list, data_collection_ other_checks = { "command_line": "Command produced by the job", + "command_version": "Tool version indicated during job execution", "stdout": "Standard output of the job", "stderr": "Standard error of the job", } @@ -1064,6 +1065,7 @@ class ToolTestDescription(object): self.output_collections = map(TestCollectionOutputDef.from_dict, processed_test_dict.get("output_collections", [])) self.command_line = processed_test_dict.get("command_line", None) + self.command_version = processed_test_dict.get("command_version", None) self.stdout = processed_test_dict.get("stdout", None) self.stderr = processed_test_dict.get("stderr", None) self.expect_exit_code = processed_test_dict.get("expect_exit_code", None) @@ -1089,6 +1091,7 @@ class ToolTestDescription(object): "output_collections": map(lambda o: o.to_dict(), self.output_collections), "num_outputs": self.num_outputs, "command_line": self.command_line, + "command_version": self.command_version, "stdout": self.stdout, "stderr": self.stderr, "expect_exit_code": self.expect_exit_code, diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 05f7355b97b..57dfedab140 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -851,6 +851,15 @@ $assertions Describe assertions about the job's standard error. +$assertions + + + + + + Describe assertions about the job's +command version. + $assertions diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 06354d91aa5..084e48fbf75 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -922,7 +922,7 @@ class Tool(Dictifiable): self.__tests = parse_tests(self, tests_source) except Exception: self.__tests = None - log.exception("Failed to parse tool tests") + log.exception("Failed to parse tool tests for tool '%s'", self.id) else: self.__tests = None self.__tests_populated = True @@ -939,7 +939,7 @@ class Tool(Dictifiable): if repo_dir.name == self.repository_name: return str(repo_dir) else: - log.error("Problem finding repository dir for tool [%s]" % self.id) + log.error("Problem finding repository dir for tool '%s'" % self.id) return repository_base_dir @@ -1128,8 +1128,7 @@ class Tool(Dictifiable): group.min = int(input_source.get("min", 0)) # Use float instead of int so that 'inf' can be used for no max group.max = float(input_source.get("max", "inf")) - assert group.min <= group.max, \ - ValueError("Min repeat count must be less-than-or-equal to the max.") + assert group.min <= group.max, ValueError("Tool with id '%s': min repeat count must be less-than-or-equal to the max." % self.id) # Force default to be within min-max range group.default = min(max(group.default, group.min), group.max) rval[group.name] = group @@ -1160,7 +1159,7 @@ class Tool(Dictifiable): test_param_input_source = input_source.parse_test_input_source() group.test_param = self.parse_param_elem(test_param_input_source, enctypes, context) if group.test_param.optional: - log.debug("Tool with id %s declares a conditional test parameter as optional, this is invalid and will be ignored." % self.id) + log.debug("Tool with id '%s': declares a conditional test parameter as optional, this is invalid and will be ignored." % self.id) group.test_param.optional = False possible_cases = list(group.test_param.legal_values) # store possible cases, undefined whens will have no inputs # Must refresh when test_param changes @@ -1174,10 +1173,10 @@ class Tool(Dictifiable): try: possible_cases.remove(case.value) except Exception: - log.debug("Tool %s: a when tag has been defined for '%s (%s) --> %s', but does not appear to be selectable." % + log.debug("Tool with id '%s': a when tag has been defined for '%s (%s) --> %s', but does not appear to be selectable." % (self.id, group.name, group.test_param.name, case.value)) for unspecified_case in possible_cases: - log.warning("Tool %s: a when tag has not been defined for '%s (%s) --> %s', assuming empty inputs." % + log.warning("Tool with id '%s': a when tag has not been defined for '%s (%s) --> %s', assuming empty inputs." % (self.id, group.name, group.test_param.name, unspecified_case)) case = ConditionalWhen() case.value = unspecified_case @@ -1231,7 +1230,7 @@ class Tool(Dictifiable): for name in param.get_dependencies(): # Let it throw exception, but give some hint what the problem might be if name not in context: - log.error("Could not find dependency '%s' of parameter '%s' in tool %s" % (name, param.name, self.name)) + log.error("Tool with id '%s': Could not find dependency '%s' of parameter '%s'" % (self.id, name, param.name)) context[name].refresh_on_change = True return param @@ -1294,7 +1293,7 @@ class Tool(Dictifiable): self.app, help_text, encoded_repository_id=self.repository_id, tool_shed_repository=self.tool_shed_repository, tool_id=self.old_id, tool_version=self.version ) except Exception: - log.exception("Exception in parse_help, so images may not be properly displayed") + log.exception("Exception in parse_help, so images may not be properly displayed for tool with id '%s'", self.id) try: self.__help = Template(rst_to_html(help_text), input_encoding='utf-8', default_filters=['decode.utf8'], @@ -1409,7 +1408,7 @@ class Tool(Dictifiable): rerun_remap_job_id = trans.app.security.decode_id(incoming['rerun_remap_job_id']) except Exception as exception: log.error(str(exception)) - raise exceptions.MessageException('Failure executing tool (attempting to rerun invalid job).') + raise exceptions.MessageException("Failure executing tool with id '%s' (attempting to rerun invalid job).", self.id) set_dataset_matcher_factory(request_context, self) @@ -1421,7 +1420,7 @@ class Tool(Dictifiable): # remap if multi-runs of tools are being used. if rerun_remap_job_id and len(expanded_incomings) > 1: raise exceptions.MessageException( - 'Failure executing tool (cannot create multiple jobs when remapping existing job).') + "Failure executing tool with id '%s' (cannot create multiple jobs when remapping existing job).", self.id) # Process incoming data validation_timer = self.app.execution_timer_factory.get_timer( @@ -1522,8 +1521,8 @@ class Tool(Dictifiable): except ToolInputsNotReadyException as e: return False, e except Exception as e: - log.exception('Exception caught while attempting tool execution:') - message = 'Error executing tool: %s' % unicodify(e) + log.exception("Exception caught while attempting to execute tool with id '%s':", self.id) + message = "Error executing tool with id '%s': %s" % (self.id, unicodify(e)) return False, message if isinstance(out_data, OrderedDict): return job, list(out_data.items()) @@ -1531,7 +1530,7 @@ class Tool(Dictifiable): if isinstance(out_data, string_types): message = out_data else: - message = 'Failure executing tool (invalid data returned from tool execution)' + message = "Failure executing tool with id '%s' (invalid data returned from tool execution)" % self.id return False, message def find_fieldstorage(self, x): diff --git a/lib/galaxy/tools/test.py b/lib/galaxy/tools/test.py index 8dc6e5d41fc..8313832d494 100644 --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -50,6 +50,7 @@ def description_from_tool_object(tool, test_index, raw_test_dict): "output_collections": raw_test_dict["output_collections"], "num_outputs": num_outputs, "command_line": raw_test_dict.get("command", None), + "command_version": raw_test_dict.get("command_version", None), "stdout": raw_test_dict.get("stdout", None), "stderr": raw_test_dict.get("stderr", None), "expect_exit_code": raw_test_dict.get("expect_exit_code", None), diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index cc0396fffd1..3c25ca9e23a 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -839,8 +839,8 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): if os.path.isdir(child_path) and recursive: self.__watch_directory(child_path, elems, integrated_elems, load_panel_dict, recursive) elif self._looks_like_a_tool(child_path): - quick_load(child_path, async_load=False) - tool_loaded = True + tool_id = quick_load(child_path, async_load=False) + tool_loaded = bool(tool_id) if (tool_loaded or force_watch) and self._tool_watcher: self._tool_watcher.watch_directory(directory, quick_load) diff --git a/lib/galaxy/tools/wrappers.py b/lib/galaxy/tools/wrappers.py index 3f1b3eabf85..2d02f791347 100644 --- a/lib/galaxy/tools/wrappers.py +++ b/lib/galaxy/tools/wrappers.py @@ -77,8 +77,11 @@ class InputValueWrapper(ToolParameterValueWrapper): if self.input.type == 'boolean' and isinstance(other, string_types): return str(self) # For backward compatibility, allow `$wrapper != ""` for optional non-text param - if self.input.optional and self.value is None and isinstance(other, string_types): - return str(self) + if self.input.optional and self.value is None: + if isinstance(other, string_types): + return str(self) + else: + return None cast = { 'text': str, 'integer': int, diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 1e488ef1c4c..1dcb20b2303 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -271,7 +271,7 @@ def xml_to_string(elem, pretty=False): except TypeError as e: # we assume this is a comment if hasattr(elem, 'text'): - return "\n" % elem.text + return u"\n" % elem.text else: raise e if xml_str and pretty: diff --git a/lib/galaxy/web/framework/webapp.py b/lib/galaxy/web/framework/webapp.py index a43cfd49ac7..273eb40cc69 100644 --- a/lib/galaxy/web/framework/webapp.py +++ b/lib/galaxy/web/framework/webapp.py @@ -563,6 +563,10 @@ class GalaxyWebTransaction(base.DefaultWebTransaction, return except IndexError: pass + authnz_controller_base = url_for(controller='authnz', action='index') + if self.request.path.startswith(authnz_controller_base): + # All authnz requests pass through + return # redirect to root if the path is not in the list above if self.request.path not in allowed_paths: login_url = url_for(controller='root', action='login', redirect=self.request.path) @@ -714,6 +718,7 @@ class GalaxyWebTransaction(base.DefaultWebTransaction, - add the disk usage of the current session to the user's total disk usage """ self.user_checks(user) + self.app.security_agent.create_user_role(user, self.app) # Set the previous session prev_galaxy_session = self.galaxy_session prev_galaxy_session.is_valid = False diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index c8ea2adc01d..20448c65260 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -21,6 +21,7 @@ from galaxy.managers.jobs import ( from galaxy.web import ( expose_api, expose_api_anonymous, + require_admin, ) from galaxy.webapps.base.controller import ( BaseAPIController, @@ -229,15 +230,13 @@ class JobController(BaseAPIController, UsesVisualizationMixin): :type id: string :param id: Encoded job id + :type message: string + :param message: Stop message. """ + payload = kwd.get("payload") or {} job = self.__get_job(trans, id) - if not job.finished: - job.mark_deleted(self.app.config.track_jobs_in_database) - trans.sa_session.flush() - self.app.job_manager.stop(job) - return True - else: - return False + message = payload.get("message", None) + return self.job_manager.stop(job, message=message) @expose_api def resume(self, trans, id, **kwd): @@ -459,3 +458,23 @@ class JobController(BaseAPIController, UsesVisualizationMixin): ) return {'messages': messages} + + @require_admin + @expose_api + def show_job_lock(self, trans, **kwd): + """ + * GET /api/job_lock + return boolean indicating if job lock active. + """ + return {"active": self.app.job_manager.job_lock} + + @require_admin + @expose_api + def update_job_lock(self, trans, payload, **kwd): + """ + * PUT /api/job_lock + return boolean indicating if job lock active. + """ + job_lock = payload.get("active") + self.app.queue_worker.send_control_task('admin_job_lock', kwargs={'job_lock': job_lock}, get_response=True) + return {"active": self.app.job_manager.job_lock} diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index f97aef719ab..0b576f60b02 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -943,6 +943,8 @@ def populate_api_routes(webapp, app): webapp.mapper.connect('common_problems', '/api/jobs/{id}/common_problems', controller='jobs', action='common_problems', conditions=dict(method=['GET'])) # Job metrics and parameters by job id or dataset id (for slightly different accessibility checking) webapp.mapper.connect('metrics', '/api/jobs/{job_id}/metrics', controller='jobs', action='metrics', conditions=dict(method=['GET'])) + webapp.mapper.connect('show_job_lock', '/api/job_lock', controller='jobs', action='show_job_lock', conditions=dict(method=['GET'])) + webapp.mapper.connect('update_job_lock', '/api/job_lock', controller='jobs', action='update_job_lock', conditions=dict(method=['PUT'])) webapp.mapper.connect('dataset_metrics', '/api/datasets/{dataset_id}/metrics', controller='jobs', action='metrics', conditions=dict(method=['GET'])) webapp.mapper.connect('parameters_display', '/api/jobs/{job_id}/parameters_display', controller='jobs', action='parameters_display', conditions=dict(method=['GET'])) webapp.mapper.connect('dataset_parameters_display', '/api/datasets/{dataset_id}/parameters_display', controller='jobs', action='parameters_display', conditions=dict(method=['GET'])) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py index 70472e83322..0ff609e5b23 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -3,7 +3,6 @@ import logging import os from collections import OrderedDict from datetime import datetime, timedelta -from string import punctuation as PUNCTUATION import six from sqlalchemy import and_, false, or_ @@ -531,6 +530,8 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP condition=(lambda item: not item.deleted and not item.purged), allow_multiple=False ) + activate_operation = grids.GridOperation("Activate User", condition=(lambda item: not item.active), allow_multiple=False) + resend_activation_email = grids.GridOperation("Resend Activation Email", condition=(lambda item: not item.active), allow_multiple=False) @web.expose @web.require_admin @@ -614,6 +615,13 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP message, status = self._recalculate_user(trans, id) elif operation == 'generate new api key': message, status = self._new_user_apikey(trans, id) + elif operation == 'activate user': + message, status = self._activate_user(trans, id) + elif operation == 'resend activation email': + message, status = self._resend_activation_email(trans, id) + if message and status: + kwd['message'] = util.sanitize_text(message) + kwd['status'] = status if trans.app.config.allow_user_deletion: if self.delete_operation not in self.user_list_grid.operations: self.user_list_grid.operations.append(self.delete_operation) @@ -624,9 +632,10 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP if trans.app.config.allow_user_impersonation: if self.impersonate_operation not in self.user_list_grid.operations: self.user_list_grid.operations.append(self.impersonate_operation) - if message and status: - kwd['message'] = util.sanitize_text(message) - kwd['status'] = status + if trans.app.config.user_activation_on: + if self.activate_operation not in self.user_list_grid.operations: + self.user_list_grid.operations.append(self.activate_operation) + self.user_list_grid.operations.append(self.resend_activation_email) return self.user_list_grid(trans, **kwd) @web.legacy_expose_api @@ -1457,6 +1466,22 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP trans.sa_session.flush() return ("New key '%s' generated for requested user '%s'." % (new_key.key, user.email), "done") + def _activate_user(self, trans, user_id): + user = trans.sa_session.query(trans.model.User).get(trans.security.decode_id(user_id)) + if not user: + return ('User not found for id (%s)' % sanitize_text(str(user_id)), 'error') + self.user_manager.activate(user) + return ('Activated user: %s.' % user.email, 'done') + + def _resend_activation_email(self, trans, user_id): + user = trans.sa_session.query(trans.model.User).get(trans.security.decode_id(user_id)) + if not user: + return ('User not found for id (%s)' % sanitize_text(str(user_id)), 'error') + if self.user_manager.send_activation_email(trans, user.email, user.username): + return ('Activation email has been sent to user: %s.' % user.email, 'done') + else: + return ('Unable to send activation email to user: %s.' % user.email, 'error') + @web.legacy_expose_api @web.require_admin def manage_roles_and_groups_for_user(self, trans, payload=None, **kwd): @@ -1508,47 +1533,9 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP @web.expose @web.json @web.require_admin - def jobs_control(self, trans, job_lock=None, **kwd): - if job_lock is not None: - job_lock = True if job_lock == 'true' else False - trans.app.queue_worker.send_control_task('admin_job_lock', kwargs={'job_lock': job_lock}, get_response=True) - job_lock = trans.app.job_manager.job_lock - return {'job_lock': job_lock} - - @web.expose - @web.json - @web.require_admin - def jobs_list(self, trans, stop=[], stop_msg=None, cutoff=180, **kwd): - deleted = [] + def jobs_list(self, trans, cutoff=180, **kwd): message = kwd.get('message', '') status = kwd.get('status', 'info') - job_ids = util.listify(stop) - if job_ids and stop_msg in [None, '']: - message = 'Please enter an error message to display to the user describing why the job was terminated' - return self.message_exception(trans, message) - elif job_ids: - if stop_msg[-1] not in PUNCTUATION: - stop_msg += '.' - for job_id in job_ids: - error_msg = "This job was stopped by an administrator: %s Contact support for additional help." \ - % (stop_msg, self.app.config.get("support_url", "https://galaxyproject.org/support/")) - if trans.app.config.track_jobs_in_database: - job = trans.sa_session.query(trans.app.model.Job).get(job_id) - job.job_stderr = error_msg - job.set_state(trans.app.model.Job.states.DELETED_NEW) - trans.sa_session.add(job) - else: - trans.app.job_manager.stop(job, message=error_msg) - deleted.append(str(job_id)) - if deleted: - message = 'Queued job' - if len(deleted) > 1: - message += 's' - message += ' for deletion: ' - message += ', '.join(deleted) - status = 'done' - trans.sa_session.flush() - job_lock = trans.app.job_manager.job_lock cutoff_time = datetime.utcnow() - timedelta(seconds=int(cutoff)) jobs = trans.sa_session.query(trans.app.model.Job) \ .filter(and_(trans.app.model.Job.table.c.update_time < cutoff_time, @@ -1566,21 +1553,15 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP def prepare_jobs_list(jobs): res = [] for job in jobs: - inputs = "" - try: - inputs = ", ".join(['{} {}'.format(da.dataset.id, da.dataset.state) for da in job.input_datasets]) - except Exception: - inputs = 'Unable to determine inputs' res.append({ 'job_info': { 'id': job.id, - 'info_url': "{}?jobid={}".format(web.url_for(controller="admin", action="job_info"), job.id) }, + 'id': trans.security.encode_id(job.id), 'user': job.history.user.email if job.history and job.history.user else 'anonymous', 'update_time': job.update_time.isoformat(), 'tool_id': job.tool_id, 'state': job.state, - 'input_dataset': inputs, 'command_line': job.command_line, 'job_runner_name': job.job_runner_name, 'job_runner_external_id': job.job_runner_external_id @@ -1590,8 +1571,7 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP 'recent_jobs': prepare_jobs_list(recent_jobs), 'cutoff': cutoff, 'message': message, - 'status': status, - 'job_lock': job_lock} + 'status': status} @web.expose @web.require_admin diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 8ceffa4a0f5..3662647a770 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -57,17 +57,6 @@ class AdminToolshed(AdminGalaxy): message=message, status=status)) - @web.expose - @web.require_admin - def browse_repository(self, trans, **kwd): - message = escape(kwd.get('message', '')) - status = kwd.get('status', 'done') - repository = repository_util.get_installed_tool_shed_repository(trans.app, kwd['id']) - return trans.fill_template('/admin/tool_shed_repository/browse_repository.mako', - repository=repository, - message=message, - status=status) - @web.legacy_expose_api @web.require_admin def browse_repositories(self, trans, **kwd): @@ -1164,27 +1153,6 @@ class AdminToolshed(AdminGalaxy): message=message, status=status) - @web.json - def tool_dependency_status_updates(self, trans, ids=None, status_list=None): - # Avoid caching - trans.response.headers['Pragma'] = 'no-cache' - trans.response.headers['Expires'] = '0' - # Create new HTML for any ToolDependency records whose status that has changed. - rval = [] - if ids is not None and status_list is not None: - ids = util.listify(ids) - status_list = util.listify(status_list) - for tup in zip(ids, status_list): - id, status = tup - tool_dependency = trans.install_model.context.query(trans.install_model.ToolDependency).get(trans.security.decode_id(id)) - if tool_dependency.status != status: - rval.append(dict(id=id, - status=tool_dependency.status, - html_status=unicodify(trans.fill_template("admin/tool_shed_repository/tool_dependency_installation_status.mako", - tool_dependency=tool_dependency), - 'utf-8'))) - return rval - @web.expose @web.require_admin def uninstall_tool_dependencies(self, trans, **kwd): @@ -1298,12 +1266,3 @@ class AdminToolshed(AdminGalaxy): id=trans.security.encode_id(repository.id), message=message, status=status)) - - @web.expose - @web.require_admin - def update_tool_shed_status_for_installed_repository(self, trans, **kwd): - message, status = repository_util.check_for_updates(trans.app, trans.install_model, kwd.get('id', None)) - return trans.response.send_redirect(web.url_for(controller='admin', - action='repositories', - message=message, - status=status)) diff --git a/lib/galaxy/webapps/galaxy/controllers/root.py b/lib/galaxy/webapps/galaxy/controllers/root.py index 49fa09f4921..6140289596b 100644 --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -87,7 +87,8 @@ class RootController(controller.JSAppLauncher, UsesAnnotations): # there is only one oidc provider, 3) auth_conf.xml has no authenticators if (trans.app.config.enable_oidc and len(trans.app.config.oidc) == 1 and - len(trans.app.auth_manager.authenticators) == 0): + len(trans.app.auth_manager.authenticators) == 0 + and not kwd.get('logout', False)): provider = trans.app.config.oidc[0] success, message, redirect_uri = trans.app.authnz_manager.authenticate(provider, trans) diff --git a/lib/galaxy/webapps/galaxy/controllers/user.py b/lib/galaxy/webapps/galaxy/controllers/user.py index 56394bcf968..2284daae3d6 100644 --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -273,9 +273,7 @@ class User(BaseUIController, UsesFormDefinitionsMixin, CreatesApiKeysMixin): return trans.show_ok_message("Your account is already active. Nothing has changed.
Go to login page.") % web.url_for(controller='root', action='index') if user.activation_token == activation_token: user.activation_token = None - user.active = True - trans.sa_session.add(user) - trans.sa_session.flush() + self.user_manager.activate(user) return trans.show_ok_message("Your account has been successfully activated!
Go to login page.") % web.url_for(controller='root', action='index') else: # Tokens don't match. Activation is denied. diff --git a/lib/galaxy_ext/container_monitor/monitor.py b/lib/galaxy_ext/container_monitor/monitor.py index b1e1447f1e0..86987b81e2d 100644 --- a/lib/galaxy_ext/container_monitor/monitor.py +++ b/lib/galaxy_ext/container_monitor/monitor.py @@ -22,7 +22,7 @@ def parse_ports(container_name, connection_configuration): preexec_fn=os.setpgrp) if exit_code == 0: stdout_file.seek(0) - ports_raw = stdout_file.read() + ports_raw = stdout_file.read().decode('utf-8') return ports_raw diff --git a/lib/galaxy_test/driver/driver_util.py b/lib/galaxy_test/driver/driver_util.py index f6334b762ee..1475c8669e1 100644 --- a/lib/galaxy_test/driver/driver_util.py +++ b/lib/galaxy_test/driver/driver_util.py @@ -4,6 +4,7 @@ import fcntl import logging import os import random +import re import shutil import signal import socket @@ -62,13 +63,15 @@ INSTALLED_TOOL_PANEL_CONFIGS = [ ] REALTIME_PROXY_TEMPLATE = string.Template(r""" uwsgi: - realtime_map: $tempdir/realtime_map.sqlite - python-raw: scripts/realtime/key_type_token_mapping.py - route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(realtime\.$test_host:$test_port)$ goto:realtime + http-raw-body: true + interactivetools_map: $tempdir/interactivetools_map.sqlite + python-raw: scripts/interactivetools/key_type_token_mapping.py + route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)-([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(interactivetool\.$test_host:$test_port)$ goto:interactivetool route-run: goto:endendend - route-label: realtime - route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(realtime\.$test_host:$test_port)$ rpcvar:TARGET_HOST rtt_key_type_token_mapper_cached $2 $1 $3 $4 $0 5 + route-label: interactivetool + route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)-([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(interactivetool\.$test_host:$test_port)$ rpcvar:TARGET_HOST rtt_key_type_token_mapper_cached $1 $3 $2 $4 $0 5 route-if-not: empty:${TARGET_HOST} httpdumb:${TARGET_HOST} + route: .* break:404 Not Found route-label: endendend """) @@ -729,8 +732,9 @@ def launch_uwsgi(kwargs, tempdir, prefix=DEFAULT_CONFIG_PREFIX, config_object=No enable_realtime_mapping = getattr(config_object, "enable_realtime_mapping", False) if enable_realtime_mapping: - config["galaxy"]["realtime_prefix"] = "realtime" - config["galaxy"]["realtime_map"] = os.path.join(tempdir, "realtime_map.sqlite") + config["galaxy"]["interactivetools_prefix"] = "interactivetool" + config["galaxy"]["interactivetools_map"] = os.path.join(tempdir, "interactivetools_map.sqlite") + config['galaxy']['interactivetools_enable'] = True yaml_config_path = os.path.join(tempdir, "galaxy.yml") with open(yaml_config_path, "w") as f: @@ -743,7 +747,7 @@ def launch_uwsgi(kwargs, tempdir, prefix=DEFAULT_CONFIG_PREFIX, config_object=No old_contents = f.read() with open(yaml_config_path, "w") as f: test_port = str(port) if port else r"[0-9]+" - test_host = host or "localhost" + test_host = re.escape(host) if host else "localhost" uwsgi_section = REALTIME_PROXY_TEMPLATE.safe_substitute(test_host=test_host, test_port=test_port, tempdir=tempdir) f.write(uwsgi_section) f.write(old_contents) diff --git a/lib/galaxy_test/driver/integration_util.py b/lib/galaxy_test/driver/integration_util.py index 37a16a33383..59e3ef17b86 100644 --- a/lib/galaxy_test/driver/integration_util.py +++ b/lib/galaxy_test/driver/integration_util.py @@ -31,7 +31,7 @@ def skip_if_jenkins(cls): def skip_unless_executable(executable): if which(executable): return _identity - return skip("PATH doesn't contain executable %s" % executable) + return pytest.mark.skip("PATH doesn't contain executable %s" % executable) def skip_unless_docker(): @@ -47,10 +47,10 @@ def k8s_config_path(): def skip_unless_fixed_port(): - if os.environ.get("GALAXY_TEST_PORT"): + if os.environ.get("GALAXY_TEST_PORT_RANDOM") != "1": return _identity - return skip("GALAXY_TEST_PORT must be set for this test.") + return pytest.mark.skip("GALAXY_TEST_PORT must be set for this test.") class IntegrationInstance(UsesApiTestCaseMixin): diff --git a/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py b/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py index 289cad2458d..b78e6d22ea2 100644 --- a/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py +++ b/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py @@ -150,10 +150,6 @@ class InstalledRepositoryGrid(grids.Grid): visible=False, filterable="standard")) global_actions = [ - grids.GridAction(label="Update tool shed status", - url_args=dict(controller='admin_toolshed', - action='update_tool_shed_status_for_installed_repository', - all_installed_repositories=True)) ] operations = [grids.GridOperation(label="Update tool shed status", condition=(lambda item: not item.deleted), diff --git a/lib/tool_shed/test/base/twilltestcase.py b/lib/tool_shed/test/base/twilltestcase.py index d5f92adbe4b..8aa315cb93d 100644 --- a/lib/tool_shed/test/base/twilltestcase.py +++ b/lib/tool_shed/test/base/twilltestcase.py @@ -1580,10 +1580,9 @@ class ShedTwillTestCase(FunctionalTestCase): self.check_for_strings(strings_displayed, strings_not_displayed) def update_tool_shed_status(self): - params = { - 'all_installed_repositories': True - } - self.visit_galaxy_url('/admin_toolshed/update_tool_shed_status_for_installed_repository', params=params) + api_key = get_master_api_key() + response = requests.get(self.galaxy_url + "/api/tool_shed_repositories/check_for_updates?key=" + api_key) + assert response.status_code != 403, response.content def upload_file(self, repository, diff --git a/lib/tool_shed/webapp/security/__init__.py b/lib/tool_shed/webapp/security/__init__.py index 17fc5bd5afe..b212e674e6e 100644 --- a/lib/tool_shed/webapp/security/__init__.py +++ b/lib/tool_shed/webapp/security/__init__.py @@ -34,6 +34,9 @@ class RBACAgent(object): """ return [x for x in [self.permitted_actions.get(action_string) for action_string in permitted_action_strings] if x is not None] + def create_user_role(self, user, app): + raise Exception("Unimplemented Method") + def create_private_user_role(self, user): raise Exception("Unimplemented Method") @@ -128,6 +131,9 @@ class CommunityRBACAgent(RBACAgent): self.associate_components(role=role, user=user) return role + def create_user_role(self, user, app): + self.get_private_user_role(user, auto_create=True) + def get_item_actions(self, action, item): # item must be one of: Dataset, Library, LibraryFolder, LibraryDataset, LibraryDatasetDatasetAssociation return [permission for permission in item.actions if permission.action == action.action] diff --git a/lib/tool_shed/webapp/templates/admin/tool_shed_repository/common.mako b/lib/tool_shed/webapp/templates/admin/tool_shed_repository/common.mako index 16f9d793f5b..fcd2ff6a9a8 100644 --- a/lib/tool_shed/webapp/templates/admin/tool_shed_repository/common.mako +++ b/lib/tool_shed/webapp/templates/admin/tool_shed_repository/common.mako @@ -258,62 +258,6 @@ %endif -<%def name="dependency_status_updater()"> - - - <%def name="repository_installation_status_updater()"> -<%def name="tool_dependency_installation_updater()"> - <% - can_update = False - if query.count(): - # Get the first tool dependency to get to the tool shed repository. - tool_dependency = query[0] - tool_shed_repository = tool_dependency.tool_shed_repository - can_update = tool_shed_repository.tool_dependencies_being_installed or tool_shed_repository.missing_tool_dependencies - %> - %if can_update: - - %endif - - <%def name="repository_installation_updater()"> <% can_update = False diff --git a/lib/tool_shed/webapp/templates/admin/tool_shed_repository/deactivate_or_uninstall_repository.mako b/lib/tool_shed/webapp/templates/admin/tool_shed_repository/deactivate_or_uninstall_repository.mako deleted file mode 100644 index 111832fea44..00000000000 --- a/lib/tool_shed/webapp/templates/admin/tool_shed_repository/deactivate_or_uninstall_repository.mako +++ /dev/null @@ -1,208 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> -<%namespace file="/admin/tool_shed_repository/common.mako" import="*" /> -<%namespace file="/admin/tool_shed_repository/repository_actions_menu.mako" import="*" /> -<%namespace file="/webapps/tool_shed/common/common.mako" import="*" /> - -<% -repository = context.get( 'repository', None ) -if isinstance( repository, list ): - repositories = repository -else: - repositories = [ repository ] -%> - -%if len( repositories ) == 1: - ${render_galaxy_repository_actions( repositories[0] )} -%endif - -%if message: - ${render_msg( message, status )} -%endif - -
-
-%for repository in repositories: - -
${repository.name|h}
-
-
- - ${repository.description|h} -
-
-
- - ${repository.changeset_revision|h} -
-
- - ${repository.tool_shed|h} -
-
-
- - ${repository.owner|h} -
-
- - ${repository.deleted|h} -
-
- <% - can_deactivate_repository = repository.can_deactivate - can_uninstall_repository = repository.can_uninstall - %> - %if can_deactivate_repository: - - - - -
- -
-
- * The repository and all of its contents will remain on disk and can still be used by dependent items. -
- %if repository.includes_tools_for_display_in_tool_panel: -
- * The repository's tools will not be loaded into the tool panel. -
- %endif - %if repository.includes_tool_dependencies: -
- * The repository's installed tool dependencies will remain on disk. -
- %endif - %if repository.includes_datatypes: -
- * The repository's datatypes, datatype converters and display applications will be eliminated from the datatypes registry. -
- %endif -
- * The repository record's deleted column in the tool_shed_repository database table will be set to True. -
-
- %endif - %if can_uninstall_repository: - - - - -
- -
-
- * The repository and all of its contents will be removed from disk and can no longer be used by dependent items. -
- %if repository.includes_tools_for_display_in_tool_panel: -
- * The repository's tool tag sets will be removed from the tool config file in which they are defined. -
- %endif - %if repository.includes_tool_dependencies: -
- * The repository's installed tool dependencies will be removed from disk and can no longer be used by dependent items. -
-
- * Each associated tool dependency record's status column in the tool_dependency database table will be set to 'Uninstalled'. -
- %endif - %if repository.includes_datatypes: -
- * The repository's datatypes, datatype converters and display applications will be eliminated from the datatypes registry. -
- %endif -
- * The repository record's deleted column in the tool_shed_repository database table will be set to True. -
-
- * The repository record's uninstalled column in the tool_shed_repository database table will be set to True. -
-
-
- <% - irm = trans.app.installed_repository_manager - repository_tup = irm.get_repository_tuple_for_installed_repository_manager( repository ) - - # Get installed repositories that this repository requires. - installed_dependent_repositories = [] - installed_runtime_dependent_tool_dependencies = [] - installed_dependent_repositories = irm.installed_dependent_repositories_of_installed_repositories.get( repository_tup, [] ) - - # Get this repository's installed tool dependencies. - installed_tool_dependencies = irm.installed_tool_dependencies_of_installed_repositories.get( repository_tup, [] ) - - # Get installed runtime dependent tool dependencies of this repository's installed tool dependencies. - installed_runtime_dependent_tool_dependencies = [] - for itd_tup in installed_tool_dependencies: - installed_dependent_td_tups = \ - irm.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies.get( itd_tup, [] ) - if installed_dependent_td_tups: - installed_runtime_dependent_tool_dependencies.extend( installed_dependent_td_tups ) - %> - %if installed_dependent_repositories or installed_runtime_dependent_tool_dependencies: - - - - -
- -
- %if installed_dependent_repositories: - -
    - %for installed_dependent_repository_tup in installed_dependent_repositories: - <% - tool_shed, name, owner, installed_changeset_revision = installed_dependent_repository_tup - %> -
  • Revision ${ installed_changeset_revision | h} of repository ${name | h} owned by ${owner | h}
    • - %endfor -
- %endif - %if installed_runtime_dependent_tool_dependencies: - -
    - %for td_tup in installed_runtime_dependent_tool_dependencies: - <% - tool_shed_repository_id, name, version, type = td_tup - containing_repository = irm.get_containing_repository_for_tool_dependency( td_tup ) - repository_name = containing_repository.name - changeset_revision = containing_repository.changeset_revision - owner = containing_repository.owner - %> -
  • - Version ${version | h} of ${type | h} ${name | h} contained in revision - ${changeset_revision | h} of repository ${repository_name | h} owned by ${owner | h} -
    • - %endfor -
- %endif -
- %endif - %endif -
-
-%endfor -
- <% - can_deactivate_repository = True in map( lambda x: x.can_deactivate, repositories ) - can_uninstall_repository = True in map( lambda x: x.can_uninstall, repositories ) - %> - %if can_deactivate_repository and can_uninstall_repository: - <% deactivate_uninstall_button_text = "Deactivate or Uninstall" %> - ${render_checkbox(remove_from_disk_check_box)} - -

- %elif can_deactivate_repository: - <% deactivate_uninstall_button_text = "Deactivate" %> - %else: - <% deactivate_uninstall_button_text = "Uninstall" %> - ##hack to mimic check box - - %endif - -
-
-
-
diff --git a/lib/tool_shed/webapp/templates/admin/tool_shed_repository/install_tool_dependencies_with_update.mako b/lib/tool_shed/webapp/templates/admin/tool_shed_repository/install_tool_dependencies_with_update.mako deleted file mode 100644 index 366eeb3bd3c..00000000000 --- a/lib/tool_shed/webapp/templates/admin/tool_shed_repository/install_tool_dependencies_with_update.mako +++ /dev/null @@ -1,94 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/admin/tool_shed_repository/repository_actions_menu.mako" import="*" /> -<%namespace file="/message.mako" import="render_msg" /> -<%namespace file="/webapps/tool_shed/common/common.mako" import="*" /> - -<% import os %> - -${render_galaxy_repository_actions( repository )} - -%if message: - ${render_msg( message, status )} -%endif - -
-

- The updates to the ${repository.name} repository require the following packages. Click the Install button to install them. - Installing some packages may take a while, but you can continue to use Galaxy during installation. -

-
- -
-
-
- - - - - - - %if tool_dependencies_dict: - %if install_tool_dependencies_check_box is not None: -
- - <% disabled = trans.app.tool_dependency_dir is None %> - ${render_checkbox(install_tool_dependencies_check_box, disabled=disabled)} -
- %if disabled: - Set the tool_dependency_dir configuration value in your Galaxy config to automatically handle tool dependencies. - %else: - Un-check to skip automatic handling of these tool dependencies. - %endif -
-
-
- %endif -
- - - - - - - - %for key, requirements_dict in tool_dependencies_dict.items(): - <% - readme_text = None - if key == 'set_environment': - key_name = ', '.join( [ environment_variable[ 'name' ] for environment_variable in requirements_dict ] ) - key_version = '' - install_dir = '' - else: - key_items = key.split( '/' ) - key_name = key_items[ 0 ] - key_version = key_items[ 1 ] - readme_text = requirements_dict.get( 'readme', None ) - install_dir = os.path.join( trans.app.tool_dependency_dir, - key_name, - key_version, - repository.owner, - repository.name, - repository.installed_changeset_revision ) - %> - %if not os.path.exists( install_dir ): - - - - - - %if readme_text: - - - %endif - %endif - %endfor -
New tool dependencies included in update
NameVersionInstall directory
${key_name|h}${key_version|h}${install_dir|h}
${key_name|h} ${key_version|h} requirements and installation information
${readme_text}
-
-
- %endif -
- -
-
-
-
diff --git a/lib/tool_shed/webapp/templates/admin/tool_shed_repository/manage_repository_tool_dependencies.mako b/lib/tool_shed/webapp/templates/admin/tool_shed_repository/manage_repository_tool_dependencies.mako deleted file mode 100644 index a1a84517339..00000000000 --- a/lib/tool_shed/webapp/templates/admin/tool_shed_repository/manage_repository_tool_dependencies.mako +++ /dev/null @@ -1,107 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> -<%namespace file="/admin/tool_shed_repository/repository_actions_menu.mako" import="*" /> -<%namespace file="/webapps/tool_shed/common/common.mako" import="*" /> - -<%def name="stylesheets()"> - ${parent.stylesheets()} - ${h.css( "library" )} - - -<%def name="javascripts()"> - ${parent.javascripts()} - ${common_misc_javascripts()} - - -${render_galaxy_repository_actions( repository )} - -%if message: - ${render_msg( message, status )} -%endif - -
-
Tool shed repository '${repository.name|h}' tool dependencies
- <% - can_install = False - can_uninstall = False - %> -

- - - %for tool_dependency in repository.tool_dependencies: - <% - if tool_dependency.error_message: - from tool_shed.util.basic_util import to_html_string - error_message = to_html_string( tool_dependency.error_message ) - else: - error_message = '' - if not can_install: - if tool_dependency.status in [ trans.install_model.ToolDependency.installation_status.NEVER_INSTALLED, - trans.install_model.ToolDependency.installation_status.UNINSTALLED ]: - can_install = True - if not can_uninstall: - if tool_dependency.status not in [ trans.install_model.ToolDependency.installation_status.NEVER_INSTALLED, - trans.install_model.ToolDependency.installation_status.UNINSTALLED ]: - can_uninstall = True - %> - - - - - - - - %endfor -
NameVersionTypeStatusError
- %if tool_dependency.status not in [ trans.install_model.ToolDependency.installation_status.UNINSTALLED ]: - - ${tool_dependency.name|h} - - %else: - ${tool_dependency.name|h} - %endif - ${tool_dependency.version|h}${tool_dependency.type|h}${tool_dependency.status|h}${error_message}
- %if can_install: -
-
-
- Check each tool dependency that you want to install and click Install. -
-
-
- Select/unselect all tool dependencies -
-
-
- ${render_select(uninstalled_tool_dependencies_select_field)} -
-
-
- -
-
-
- %endif - %if can_uninstall: -
-
-
- Check each tool dependency that you want to uninstall and click Uninstall. -
-
-
- Select/unselect all tool dependencies -
-
-
- ${render_select(installed_tool_dependencies_select_field)} -
-
-
- -
-
-
- %endif -
- diff --git a/lib/tool_shed/webapp/templates/admin/tool_shed_repository/monitor_repository_installation.mako b/lib/tool_shed/webapp/templates/admin/tool_shed_repository/monitor_repository_installation.mako deleted file mode 100644 index e42c3f034c7..00000000000 --- a/lib/tool_shed/webapp/templates/admin/tool_shed_repository/monitor_repository_installation.mako +++ /dev/null @@ -1,70 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> -<%namespace file="/admin/tool_shed_repository/common.mako" import="*" /> - -<%def name="javascripts()"> - ${parent.javascripts()} - ${repository_installation_status_updater()} - ${repository_installation_updater()} - ${self.repository_installation_javascripts()} - - -<%def name="repository_installation_javascripts()"> - - - -%if context.get("message"): - ${render_msg( message, status )} -%endif - -%if tool_shed_repositories: -
-
Monitor installing tool shed repositories
-
- - - - - - - - - %for tool_shed_repository in tool_shed_repositories: - <% - encoded_repository_id = trans.security.encode_id( tool_shed_repository.id ) - ids_of_tool_dependencies_missing_or_being_installed = [ trans.security.encode_id( td.id ) for td in tool_shed_repository.tool_dependencies_missing_or_being_installed ] - link_to_manage_tool_dependencies = tool_shed_repository.status in [ trans.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ] - %> - - - - - - - - %endfor -
NameDescriptionOwnerRevisionStatus
${tool_shed_repository.name|h}${tool_shed_repository.description}${tool_shed_repository.owner}${tool_shed_repository.changeset_revision}
${tool_shed_repository.status|h}
-
-
-
-%else: - ${render_msg("There is no repository being installed.", "info")} -%endif diff --git a/lib/tool_shed/webapp/templates/admin/tool_shed_repository/purge_repository_confirmation.mako b/lib/tool_shed/webapp/templates/admin/tool_shed_repository/purge_repository_confirmation.mako deleted file mode 100644 index 36a7f562b66..00000000000 --- a/lib/tool_shed/webapp/templates/admin/tool_shed_repository/purge_repository_confirmation.mako +++ /dev/null @@ -1,74 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> -<%namespace file="/admin/tool_shed_repository/repository_actions_menu.mako" import="*" /> - -<%def name="stylesheets()"> - ${parent.stylesheets()} - ${h.css( "library" )} - - -<%def name="javascripts()"> - ${parent.javascripts()} - - -${render_galaxy_repository_actions( repository )} - -%if message: - ${render_msg( message, status )} -%endif - -
-

- Purging the repository named ${repository.name|h} will result in deletion of all records for the - following associated items from the database. Click the Purge button to purge this repository - and its associated items. -

-
- -
-
Purge tool shed repository ${repository.name|h}
-
- <% - tool_versions = 0 - tool_dependencies = 0 - required_repositories = 0 - orphan_repository_repository_dependency_association_records = 0 - orphan_repository_dependency_records = 0 - # Count this repository's tool version lineage chain links that will be purged. - for tool_version in repository.tool_versions: - for tool_version_association in tool_version.parent_tool_association: - tool_versions += 1 - for tool_version_association in tool_version.child_tool_association: - tool_versions += 1 - tool_versions += 1 - # Count this repository's associated tool dependencies that will be purged. - for tool_dependency in repository.tool_dependencies: - tool_dependencies += 1 - # Count this repository's associated required repositories that will be purged. - for rrda in repository.required_repositories: - required_repositories += 1 - # Count any "orphan" repository_dependency records associated with the repository but not with any - # repository_repository_dependency_association records that will be purged. - for orphan_repository_dependency in \ - trans.sa_session.query( trans.app.install_model.RepositoryDependency ) \ - .filter( trans.app.install_model.RepositoryDependency.table.c.tool_shed_repository_id == repository.id ): - for orphan_rrda in \ - trans.sa_session.query( trans.app.install_model.RepositoryRepositoryDependencyAssociation ) \ - .filter( trans.app.install_model.RepositoryRepositoryDependencyAssociation.table.c.repository_dependency_id == orphan_repository_dependency.id ): - orphan_repository_repository_dependency_association_records += 1 - orphan_repository_dependency_records += 1 - %> - - - - - - -
Tool version records${tool_versions|h}
Tool dependency records${tool_dependencies|h}
Repository dependency records${required_repositories|h}
Orphan repository_repository_dependency_association records${orphan_repository_repository_dependency_association_records|h}
Orphan repository_dependency records${orphan_repository_dependency_records|h}
-
-
- -
-
-
- diff --git a/lib/tool_shed/webapp/templates/admin/tool_shed_repository/reselect_tool_panel_section.mako b/lib/tool_shed/webapp/templates/admin/tool_shed_repository/reselect_tool_panel_section.mako deleted file mode 100644 index d901b6d08e8..00000000000 --- a/lib/tool_shed/webapp/templates/admin/tool_shed_repository/reselect_tool_panel_section.mako +++ /dev/null @@ -1,101 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> -<%namespace file="/admin/tool_shed_repository/common.mako" import="render_dependencies_section" /> -<%namespace file="/admin/tool_shed_repository/common.mako" import="render_readme_section" /> -<%namespace file="/webapps/tool_shed/repository/common.mako" import="*" /> -<%namespace file="/webapps/tool_shed/common/common.mako" import="*" /> - -<%def name="stylesheets()"> - ${parent.stylesheets()} - ${h.css( "library" )} - - -<%def name="javascripts()"> - ${parent.javascripts()} - ${container_javascripts()} - - -%if message: - ${render_msg( message, status )} -%endif - -
-
-
-
- -
-
- <% readme_files_dict = containers_dict.get( 'readme_files', None ) %> - %if readme_files_dict: -
- - -
Repository README files - may contain important installation or license information
-
- ${render_readme_section( containers_dict )} -
- %endif - %if has_repository_dependencies or includes_tool_dependencies: -
- - -
Confirm dependency installation
-
- ${render_dependencies_section( install_resolver_dependencies_check_box, install_repository_dependencies_check_box, install_tool_dependencies_check_box, containers_dict, revision_label=None, export=False )} - %endif - %if shed_tool_conf_select_field: -
- - -
Choose the configuration file whose tool_path setting will be used for installing repositories
-
- <% - if len( shed_tool_conf_select_field.options ) == 1: - select_help = "Your Galaxy instance is configured with 1 shed-related tool configuration file, so repositories will be " - select_help += "installed using its tool_path setting." - else: - select_help = "Your Galaxy instance is configured with %d shed-related tool configuration files, " % len( shed_tool_conf_select_field.options ) - select_help += "so select the file whose tool_path setting you want used for installing repositories." - %> -
- - ${render_select(shed_tool_conf_select_field)} -
- ${select_help|h} -
-
-
- %else: - - %endif - %if includes_tools_for_display_in_tool_panel: -
-
- ${render_checkbox(no_changes_check_box)} - -
- Uncheck and select a different tool panel section to load the tools into a different section in the tool panel. -
-
-
- - -
- Add a new tool panel section to contain the installed tools (optional). -
-
-
- - ${render_select(tool_panel_section_select_field)} -
- Choose an existing section in your tool panel to contain the installed tools (optional). -
-
- %endif -
- -
-
-
-
diff --git a/lib/tool_shed/webapp/templates/admin/tool_shed_repository/uninstall_tool_dependencies.mako b/lib/tool_shed/webapp/templates/admin/tool_shed_repository/uninstall_tool_dependencies.mako deleted file mode 100644 index 321d5f708d2..00000000000 --- a/lib/tool_shed/webapp/templates/admin/tool_shed_repository/uninstall_tool_dependencies.mako +++ /dev/null @@ -1,63 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> -<%namespace file="/admin/tool_shed_repository/repository_actions_menu.mako" import="*" /> - -<% import os %> - -${render_galaxy_repository_actions( repository )} - -%if message: - ${render_msg( message, status )} -%endif - -
-
Uninstall tool dependencies
-
-
-
- - - - - - - - %for tool_dependency in tool_dependencies: - - <% - if tool_dependency.type == 'package': - install_dir = os.path.join( trans.app.tool_dependency_dir, - tool_dependency.name, - tool_dependency.version, - tool_dependency.tool_shed_repository.owner, - tool_dependency.tool_shed_repository.name, - tool_dependency.tool_shed_repository.installed_changeset_revision ) - elif tool_dependency.type == 'set_environment': - install_dir = os.path.join( trans.app.tool_dependency_dir, - 'environment_settings', - tool_dependency.name, - tool_dependency.tool_shed_repository.owner, - tool_dependency.tool_shed_repository.name, - tool_dependency.tool_shed_repository.installed_changeset_revision ) - if not os.path.exists( install_dir ): - install_dir = "This dependency's installation directory does not exist, click Uninstall to reset for installation." - %> - - - - - - - %endfor -
NameVersionTypeInstall directory
${tool_dependency.name|h}${tool_dependency.version|h}${tool_dependency.type|h}${install_dir|h}
-
-
-
- -
- Click to uninstall the tool dependencies listed above. -
-
-
-
-
diff --git a/packages/tool_util/HISTORY.rst b/packages/tool_util/HISTORY.rst index 52c2df7066e..cec433c3e50 100644 --- a/packages/tool_util/HISTORY.rst +++ b/packages/tool_util/HISTORY.rst @@ -6,7 +6,13 @@ History .. to_doc --------------------- -19.9.1.dev0 +19.9.2.dev0 +--------------------- + + + +--------------------- +19.9.1 (2019-12-28) --------------------- diff --git a/packages/tool_util/galaxy/project_galaxy_tool_util.py b/packages/tool_util/galaxy/project_galaxy_tool_util.py index 0ee2dfac196..c9c01454c57 100644 --- a/packages/tool_util/galaxy/project_galaxy_tool_util.py +++ b/packages/tool_util/galaxy/project_galaxy_tool_util.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__version__ = '19.9.1.dev0' +__version__ = '19.9.2.dev0' PROJECT_NAME = "galaxy-tool-util" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" diff --git a/run_tests.sh b/run_tests.sh index 78ca1b2e911..77c7871043c 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -602,6 +602,7 @@ if [ -z "$skip_common_startup" ]; then export GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION fi ./scripts/common_startup.sh $skip_venv $no_create_venv $no_replace_pip $replace_pip $skip_client_build --dev-wheels || exit 1 + unset GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION fi . ./scripts/common_startup_functions.sh diff --git a/scripts/grt/export.py b/scripts/grt/export.py index 32dc4641ddb..a782e4c8e34 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -146,150 +146,146 @@ def main(argv): blacklisted_tools = config['sanitization']['tools'] annotate('export_jobs_start', 'Exporting Jobs') - handle_job = io.open(REPORT_BASE + '.jobs.tsv', 'w', encoding='utf-8') - handle_job.write('\t'.join(('id', 'tool_id', 'tool_version', 'state', 'create_time')) + '\n') - for offset_start in range(last_job_sent, end_job_id, args.batch_size): - logging.debug("Processing %s:%s", offset_start, min(end_job_id, offset_start + args.batch_size)) - for job in sa_session.query(model.Job.id, model.Job.user_id, model.Job.tool_id, model.Job.tool_version, model.Job.state, model.Job.create_time) \ - .filter(model.Job.id > offset_start) \ - .filter(model.Job.id <= min(end_job_id, offset_start + args.batch_size)) \ - .all(): - # If the tool is blacklisted, exclude everywhere - if job[2] in blacklisted_tools: - continue + with io.open(REPORT_BASE + '.jobs.tsv', 'w', encoding='utf-8') as handle_job: + handle_job.write(u'\t'.join(('id', 'tool_id', 'tool_version', 'state', 'create_time')) + '\n') + for offset_start in range(last_job_sent, end_job_id, args.batch_size): + logging.debug("Processing %s:%s", offset_start, min(end_job_id, offset_start + args.batch_size)) + for job in sa_session.query(model.Job.id, model.Job.user_id, model.Job.tool_id, model.Job.tool_version, model.Job.state, model.Job.create_time) \ + .filter(model.Job.id > offset_start) \ + .filter(model.Job.id <= min(end_job_id, offset_start + args.batch_size)) \ + .all(): + # If the tool is blacklisted, exclude everywhere + if job[2] in blacklisted_tools: + continue - try: - line = [ - str(job[0]), # id - job[2], # tool_id - job[3], # tool_version - job[4], # state - str(job[5]) # create_time - ] - cline = unicodify('\t'.join(line) + '\n') - handle_job.write(cline) - except Exception: - logging.warning("Unable to write out a 'handle_job' row. Ignoring the row.", exc_info=True) - continue - # meta counts - job_state_data[job[4]] += 1 - active_users[job[1]] += 1 - job_tool_map[job[0]] = job[2] - - handle_job.close() + try: + line = [ + str(job[0]), # id + job[2], # tool_id + job[3], # tool_version + job[4], # state + str(job[5]) # create_time + ] + cline = unicodify('\t'.join(line) + '\n') + handle_job.write(cline) + except Exception: + logging.warning("Unable to write out a 'handle_job' row. Ignoring the row.", exc_info=True) + continue + # meta counts + job_state_data[job[4]] += 1 + active_users[job[1]] += 1 + job_tool_map[job[0]] = job[2] annotate('export_jobs_end') annotate('export_datasets_start', 'Exporting Datasets') - handle_datasets = io.open(REPORT_BASE + '.datasets.tsv', 'w', encoding='utf-8') - handle_datasets.write('\t'.join(('job_id', 'dataset_id', 'extension', 'file_size', 'param_name', 'type')) + '\n') - for offset_start in range(last_job_sent, end_job_id, args.batch_size): - logging.debug("Processing %s:%s", offset_start, min(end_job_id, offset_start + args.batch_size)) + with io.open(REPORT_BASE + '.datasets.tsv', 'w', encoding='utf-8') as handle_datasets: + handle_datasets.write(u'\t'.join(('job_id', 'dataset_id', 'extension', 'file_size', 'param_name', 'type')) + '\n') + for offset_start in range(last_job_sent, end_job_id, args.batch_size): + logging.debug("Processing %s:%s", offset_start, min(end_job_id, offset_start + args.batch_size)) - # four queries: JobToInputDatasetAssociation, JobToOutputDatasetAssociation, HistoryDatasetAssociation, Dataset + # four queries: JobToInputDatasetAssociation, JobToOutputDatasetAssociation, HistoryDatasetAssociation, Dataset - job_to_input_hda_ids = sa_session.query(model.JobToInputDatasetAssociation.job_id, model.JobToInputDatasetAssociation.dataset_id, - model.JobToInputDatasetAssociation.name) \ - .filter(model.JobToInputDatasetAssociation.job_id > offset_start) \ - .filter(model.JobToInputDatasetAssociation.job_id <= min(end_job_id, offset_start + args.batch_size)) \ - .all() + job_to_input_hda_ids = sa_session.query(model.JobToInputDatasetAssociation.job_id, model.JobToInputDatasetAssociation.dataset_id, + model.JobToInputDatasetAssociation.name) \ + .filter(model.JobToInputDatasetAssociation.job_id > offset_start) \ + .filter(model.JobToInputDatasetAssociation.job_id <= min(end_job_id, offset_start + args.batch_size)) \ + .all() - job_to_output_hda_ids = sa_session.query(model.JobToOutputDatasetAssociation.job_id, model.JobToOutputDatasetAssociation.dataset_id, - model.JobToOutputDatasetAssociation.name) \ - .filter(model.JobToOutputDatasetAssociation.job_id > offset_start) \ - .filter(model.JobToOutputDatasetAssociation.job_id <= min(end_job_id, offset_start + args.batch_size)) \ - .all() + job_to_output_hda_ids = sa_session.query(model.JobToOutputDatasetAssociation.job_id, model.JobToOutputDatasetAssociation.dataset_id, + model.JobToOutputDatasetAssociation.name) \ + .filter(model.JobToOutputDatasetAssociation.job_id > offset_start) \ + .filter(model.JobToOutputDatasetAssociation.job_id <= min(end_job_id, offset_start + args.batch_size)) \ + .all() - # add type and concat - job_to_hda_ids = [[list(i), "input"] for i in job_to_input_hda_ids] + [[list(i), "output"] for i in job_to_output_hda_ids] + # add type and concat + job_to_hda_ids = [[list(i), "input"] for i in job_to_input_hda_ids] + [[list(i), "output"] for i in job_to_output_hda_ids] - # put all of the hda_ids into a list - hda_ids = [i[0][1] for i in job_to_hda_ids] + # put all of the hda_ids into a list + hda_ids = [i[0][1] for i in job_to_hda_ids] - hdas = sa_session.query(model.HistoryDatasetAssociation.id, model.HistoryDatasetAssociation.dataset_id, - model.HistoryDatasetAssociation.extension) \ - .filter(model.HistoryDatasetAssociation.id.in_(hda_ids)) \ - .all() + hdas = sa_session.query(model.HistoryDatasetAssociation.id, model.HistoryDatasetAssociation.dataset_id, + model.HistoryDatasetAssociation.extension) \ + .filter(model.HistoryDatasetAssociation.id.in_(hda_ids)) \ + .all() - # put all the dataset ids into a list - dataset_ids = [i[1] for i in hdas] + # put all the dataset ids into a list + dataset_ids = [i[1] for i in hdas] - # get the sizes of the datasets - datasets = sa_session.query(model.Dataset.id, model.Dataset.total_size) \ - .filter(model.Dataset.id.in_(dataset_ids)) \ - .all() + # get the sizes of the datasets + datasets = sa_session.query(model.Dataset.id, model.Dataset.total_size) \ + .filter(model.Dataset.id.in_(dataset_ids)) \ + .all() - # datasets to dictionay for easy search - hdas = {i[0]: i[1:] for i in hdas} - datasets = {i[0]: i[1:] for i in datasets} + # datasets to dictionay for easy search + hdas = {i[0]: i[1:] for i in hdas} + datasets = {i[0]: i[1:] for i in datasets} - for job_to_hda in job_to_hda_ids: + for job_to_hda in job_to_hda_ids: - job = job_to_hda[0] # job_id, hda_id, name - filetype = job_to_hda[1] # input|output + job = job_to_hda[0] # job_id, hda_id, name + filetype = job_to_hda[1] # input|output - # No associated job - if job[0] not in job_tool_map: - continue + # No associated job + if job[0] not in job_tool_map: + continue - # If the tool is blacklisted, exclude everywhere - if job_tool_map[job[0]] in blacklisted_tools: - continue + # If the tool is blacklisted, exclude everywhere + if job_tool_map[job[0]] in blacklisted_tools: + continue - hda_id = job[1] - if hda_id is None: - continue + hda_id = job[1] + if hda_id is None: + continue - dataset_id = hdas[hda_id][0] - if dataset_id is None: - continue + dataset_id = hdas[hda_id][0] + if dataset_id is None: + continue - try: - line = [ - str(job[0]), # Job ID - str(hda_id), # HDA ID - str(hdas[hda_id][1]), # Extension - round_to_2sd(datasets[dataset_id][0]), # File size - job[2], # Parameter name - str(filetype) # input/output - ] - cline = unicodify('\t'.join(line) + '\n') - handle_datasets.write(cline) - except Exception: - logging.warning("Unable to write out a 'handle_datasets' row. Ignoring the row.", exc_info=True) - continue - handle_datasets.close() + try: + line = [ + str(job[0]), # Job ID + str(hda_id), # HDA ID + str(hdas[hda_id][1]), # Extension + round_to_2sd(datasets[dataset_id][0]), # File size + job[2], # Parameter name + str(filetype) # input/output + ] + cline = unicodify('\t'.join(line) + '\n') + handle_datasets.write(cline) + except Exception: + logging.warning("Unable to write out a 'handle_datasets' row. Ignoring the row.", exc_info=True) + continue annotate('export_datasets_end') annotate('export_metric_num_start', 'Exporting Metrics (Numeric)') - handle_metric_num = io.open(REPORT_BASE + '.metric_num.tsv', 'w', encoding='utf-8') - handle_metric_num.write('\t'.join(('job_id', 'plugin', 'name', 'value')) + '\n') - for offset_start in range(last_job_sent, end_job_id, args.batch_size): - logging.debug("Processing %s:%s", offset_start, min(end_job_id, offset_start + args.batch_size)) - for metric in sa_session.query(model.JobMetricNumeric.job_id, model.JobMetricNumeric.plugin, model.JobMetricNumeric.metric_name, model.JobMetricNumeric.metric_value) \ - .filter(model.JobMetricNumeric.job_id > offset_start) \ - .filter(model.JobMetricNumeric.job_id <= min(end_job_id, offset_start + args.batch_size)) \ - .all(): - # No associated job - if metric[0] not in job_tool_map: - continue - # If the tool is blacklisted, exclude everywhere - if job_tool_map[metric[0]] in blacklisted_tools: - continue + with io.open(REPORT_BASE + '.metric_num.tsv', 'w', encoding='utf-8') as handle_metric_num: + handle_metric_num.write(u'\t'.join(('job_id', 'plugin', 'name', 'value')) + '\n') + for offset_start in range(last_job_sent, end_job_id, args.batch_size): + logging.debug("Processing %s:%s", offset_start, min(end_job_id, offset_start + args.batch_size)) + for metric in sa_session.query(model.JobMetricNumeric.job_id, model.JobMetricNumeric.plugin, model.JobMetricNumeric.metric_name, model.JobMetricNumeric.metric_value) \ + .filter(model.JobMetricNumeric.job_id > offset_start) \ + .filter(model.JobMetricNumeric.job_id <= min(end_job_id, offset_start + args.batch_size)) \ + .all(): + # No associated job + if metric[0] not in job_tool_map: + continue + # If the tool is blacklisted, exclude everywhere + if job_tool_map[metric[0]] in blacklisted_tools: + continue - try: - line = [ - str(metric[0]), # job id - metric[1], # plugin - metric[2], # name - str(metric[3]) # value - ] + try: + line = [ + str(metric[0]), # job id + metric[1], # plugin + metric[2], # name + str(metric[3]) # value + ] - cline = unicodify('\t'.join(line) + '\n') - handle_metric_num.write(cline) - except Exception: - logging.warning("Unable to write out a 'handle_metric_num' row. Ignoring the row.", exc_info=True) - continue - handle_metric_num.close() + cline = unicodify('\t'.join(line) + '\n') + handle_metric_num.write(cline) + except Exception: + logging.warning("Unable to write out a 'handle_metric_num' row. Ignoring the row.", exc_info=True) + continue annotate('export_metric_num_end') # Now on to outputs. diff --git a/scripts/interactivetools/key_type_token_mapping.py b/scripts/interactivetools/key_type_token_mapping.py index fa096dfbd60..df6dff6506a 100644 --- a/scripts/interactivetools/key_type_token_mapping.py +++ b/scripts/interactivetools/key_type_token_mapping.py @@ -6,7 +6,7 @@ from time import time import uwsgi -realtime_db_file = uwsgi.opt["interactivetools_map"] +realtime_db_file = uwsgi.opt["interactivetools_map"].decode('utf-8') db_conn = sqlite3.connect(realtime_db_file) DATABASE_TABLE_NAME = 'gxitproxy' @@ -51,6 +51,14 @@ class CacheList(): key_type_token_mapped_cache = CacheList() +def args_as_unicode(func): + def wrap_args(*args): + args = (arg.decode('utf-8') if isinstance(arg, bytes) else arg for arg in args) + return func(*args) + return wrap_args + + +@args_as_unicode def key_type_token_mapper_cached(key, key_type, token, route_extra, url, ttl): global key_type_token_mapped_cache cache_key = (key, key_type, token) @@ -58,11 +66,12 @@ def key_type_token_mapper_cached(key, key_type, token, route_extra, url, ttl): if entry is None: entry = key_type_token_mapper(key, key_type, token, route_extra, url) if entry is not None: - # Should we cache empt/not authorized entries, perhaps for shorter time? + # Should we cache empty/not authorized entries, perhaps for shorter time? key_type_token_mapped_cache.add_entry(cache_key, entry, ttl=float(ttl)) return entry +@args_as_unicode def key_type_token_mapper(key, key_type, token, route_extra, url): global db_conn # print 'key %s key_type %s token %s route_extra %s url %s\n' % (key, key_type, token, route_extra, url) diff --git a/templates/admin/tool_shed_repository/browse_repository.mako b/templates/admin/tool_shed_repository/browse_repository.mako deleted file mode 100644 index 0931cd34f54..00000000000 --- a/templates/admin/tool_shed_repository/browse_repository.mako +++ /dev/null @@ -1,36 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> -<%namespace file="/admin/tool_shed_repository/common.mako" import="*" /> -<%namespace file="/admin/tool_shed_repository/repository_actions_menu.mako" import="*" /> - -<%def name="stylesheets()"> - ${parent.stylesheets()} - ${h.css( "dynatree_skin/ui.dynatree" )} - - -<%def name="javascripts()"> - ${parent.javascripts()} - ## ${h.js( "libs/jquery/jquery-ui", "libs/jquery/jquery.dynatree" )} - ${browse_files(repository.name, repository.repo_files_directory(trans.app))} - - -${render_galaxy_repository_actions( repository )} - -%if message: - ${render_msg( message, status )} -%endif - -
-
Browse ${repository.name|h} revision ${repository.changeset_revision} files
-
-
- -
- Loading... -
-
-
-
-
-
-
diff --git a/templates/admin/tool_shed_repository/common.mako b/templates/admin/tool_shed_repository/common.mako index 16f9d793f5b..fcd2ff6a9a8 100644 --- a/templates/admin/tool_shed_repository/common.mako +++ b/templates/admin/tool_shed_repository/common.mako @@ -258,62 +258,6 @@ %endif -<%def name="dependency_status_updater()"> - - - <%def name="repository_installation_status_updater()"> -<%def name="tool_dependency_installation_updater()"> - <% - can_update = False - if query.count(): - # Get the first tool dependency to get to the tool shed repository. - tool_dependency = query[0] - tool_shed_repository = tool_dependency.tool_shed_repository - can_update = tool_shed_repository.tool_dependencies_being_installed or tool_shed_repository.missing_tool_dependencies - %> - %if can_update: - - %endif - - <%def name="repository_installation_updater()"> <% can_update = False diff --git a/test/functional/tools/job_properties.xml b/test/functional/tools/job_properties.xml index 1bad9a7f4e2..753aa026a25 100644 --- a/test/functional/tools/job_properties.xml +++ b/test/functional/tools/job_properties.xml @@ -2,6 +2,7 @@ + echo 'v1.1' + + + diff --git a/test/integration/test_cli_runners.py b/test/integration/test_cli_runners.py index d09e6c07fd3..7ac42b5f333 100644 --- a/test/integration/test_cli_runners.py +++ b/test/integration/test_cli_runners.py @@ -3,6 +3,7 @@ import collections import os import string import subprocess +import sys import tempfile import unittest @@ -11,11 +12,9 @@ from galaxy_test.base.ssh_util import generate_ssh_keys from galaxy_test.driver import integration_util from .test_job_environments import BaseJobEnvironmentIntegrationTestCase - -RemoteConnection = collections.namedtuple('remote_connection', ['hostname', 'username', 'password', 'port', 'private_key', 'public_key']) +RemoteConnection = collections.namedtuple('remote_connection', ['hostname', 'username', 'port', 'private_key', 'public_key']) -@integration_util.skip_unless_docker() def start_ssh_docker(container_name, jobs_directory, port=10022, image='agaveapi/slurm'): ssh_keys = generate_ssh_keys() START_SLURM_DOCKER = ['docker', @@ -28,6 +27,7 @@ def start_ssh_docker(container_name, jobs_directory, port=10022, image='agaveapi '--name', container_name, '--rm', + '--privileged', # for torque '-v', "{jobs_directory}:{jobs_directory}".format(jobs_directory=jobs_directory), "-v", @@ -36,7 +36,9 @@ def start_ssh_docker(container_name, jobs_directory, port=10022, image='agaveapi 'nofile=2048:2048', image] subprocess.check_call(START_SLURM_DOCKER) - return RemoteConnection('localhost', 'testuser', 'testuser', port, ssh_keys.private_key_file, ssh_keys.public_key_file) + if sys.platform != 'darwin': + subprocess.check_call(['docker', 'exec', container_name, 'usermod', '-u', str(os.getuid()), 'testuser']) + return RemoteConnection('localhost', 'testuser', port, ssh_keys.private_key_file, ssh_keys.public_key_file) def stop_ssh_docker(container_name, remote_connection): @@ -58,6 +60,7 @@ def cli_job_config(remote_connection, shell_plugin='ParamikoShell', job_plugin=' $private_key $hostname $port + False False 42 @@ -72,6 +75,7 @@ def cli_job_config(remote_connection, shell_plugin='ParamikoShell', job_plugin=' return job_conf.name +@integration_util.skip_unless_docker() class BaseCliIntegrationTestCase(BaseJobEnvironmentIntegrationTestCase): @classmethod @@ -91,7 +95,7 @@ class BaseCliIntegrationTestCase(BaseJobEnvironmentIntegrationTestCase): super(BaseCliIntegrationTestCase, cls).tearDownClass() @classmethod - def handle_galaxy_config_kwds(cls, config, ): + def handle_galaxy_config_kwds(cls, config): config["jobs_directory"] = cls.jobs_directory config["file_path"] = cls.jobs_directory config["job_config_file"] = cli_job_config(remote_connection=cls.remote_connection, diff --git a/test/integration/test_containerized_jobs.py b/test/integration/test_containerized_jobs.py index 626d63c5e63..6d6791e8ce8 100644 --- a/test/integration/test_containerized_jobs.py +++ b/test/integration/test_containerized_jobs.py @@ -1,5 +1,6 @@ """Integration tests for running tools in Docker containers.""" +import json import os import unittest @@ -37,6 +38,11 @@ class ContainerizedIntegrationTestCase(integration_util.IntegrationTestCase): skip_if_container_type_unavailable(cls) super(ContainerizedIntegrationTestCase, cls).setUpClass() + @classmethod + def handle_galaxy_config_kwds(cls, config): + config["job_config_file"] = DOCKERIZED_JOB_CONFIG_FILE + disable_dependency_resolution(config) + def disable_dependency_resolution(config): # Disable tool dependency resolution. @@ -56,7 +62,6 @@ class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, Ru job_config_file = DOCKERIZED_JOB_CONFIG_FILE build_mulled_resolver = 'build_mulled' container_type = 'docker' - default_container_home_dir = '/' @classmethod def handle_galaxy_config_kwds(cls, config): @@ -100,19 +105,24 @@ class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, Ru assert job_env.pwd.endswith("/working") # Should we change env_pass_through to just always include TMP and HOME for docker? # I'm not sure, if yes this would change. - assert job_env.home == self.default_container_home_dir, job_env.home + assert not job_env.home.endswith('/home') def test_build_mulled(self): if not which('docker'): raise unittest.SkipTest("Docker not found on PATH, required for building images via involucro") resolver_type = self.build_mulled_resolver - tool_id = 'mulled_example_multi_1' - endpoint = "tools/%s/dependencies" % tool_id - data = {'id': tool_id, 'resolver_type': resolver_type} + tool_ids = ['mulled_example_multi_1'] + endpoint = "dependency_resolvers/toolbox/install" + data = {'tool_ids': json.dumps(tool_ids), 'resolver_type': resolver_type, 'container_type': self.container_type, 'include_containers': True} create_response = self._post(endpoint, data=data, admin=True) self._assert_status_code_is(create_response, 200) + create_response = self._get("dependency_resolvers/toolbox", data={'tool_ids': tool_ids, 'container_type': self.container_type, 'include_containers': True, 'index_by': 'tools'}, admin=True) response = create_response.json() - assert any([True for d in response if d['dependency_type'] == self.container_type]) + assert len(response) == 1 + status = response[0]['status'] + assert status[0]['model_class'] == 'ContainerDependency' + assert status[0]['dependency_type'] == self.container_type + assert status[0]['container_description']['identifier'].startswith('quay.io/local/mulled-v2-') class MappingContainerResolverTestCase(integration_util.IntegrationTestCase): @@ -164,5 +174,3 @@ class SingularityJobsIntegrationTestCase(DockerizedJobsIntegrationTestCase): job_config_file = SINGULARITY_JOB_CONFIG_FILE build_mulled_resolver = 'build_mulled_singularity' container_type = 'singularity' - # singularity passes $HOME by default - default_container_home_dir = os.environ.get('HOME', '/') diff --git a/test/integration/test_interactivetools_api.py b/test/integration/test_interactivetools_api.py index e214e3eee80..f3ae60d4879 100644 --- a/test/integration/test_interactivetools_api.py +++ b/test/integration/test_interactivetools_api.py @@ -47,8 +47,7 @@ class InteractiveToolsIntegrationTestCase(ContainerizedIntegrationTestCase): assert isinstance(jobs, list) assert len(jobs) == 1 job0 = jobs[0] - entry_points = self.wait_on_entry_points_active(job0["id"]) - assert len(entry_points) == 2 + entry_points = self.wait_on_entry_points_active(job0["id"], expected_num=2) entry_point0 = entry_points[0] entry_point1 = entry_points[1] target0 = self.entry_point_target(entry_point0["id"]) @@ -59,19 +58,18 @@ class InteractiveToolsIntegrationTestCase(ContainerizedIntegrationTestCase): content1 = self.wait_on_proxied_content(target1) assert content1 == "moo cow\n", content1 - assert False def wait_on_proxied_content(self, target): def get_hosted_content(): try: scheme, rest = target.split("://", 1) - prefix, host_and_port = rest.split(".realtime.") - print(rest) + prefix, host_and_port = rest.split(".interactivetool.") faked_host = rest if "/" in rest: faked_host = rest.split("/", 1)[0] - response = requests.get("%s://%s" % (scheme, host_and_port), timeout=1, headers={"Host": faked_host}) - return response.content + url = "%s://%s" % (scheme, host_and_port) + response = requests.get(url, timeout=1, headers={"Host": faked_host}) + return response.text except Exception as e: print(e) return None diff --git a/test/integration/test_kubernetes_runner.py b/test/integration/test_kubernetes_runner.py index 69966b6870c..62fe4640a82 100644 --- a/test/integration/test_kubernetes_runner.py +++ b/test/integration/test_kubernetes_runner.py @@ -1,5 +1,6 @@ """Integration tests for the Kubernetes runner.""" -# Tested on docker for mac 18.06.1-ce-mac73 using the default kubernetes setup +# Tested on docker for mac 18.06.1-ce-mac73 using the default kubernetes setup, +# also works on minikube import collections import json import os @@ -91,37 +92,40 @@ def job_config(jobs_directory): jobs-directory-claim:$jobs_directory,tool-directory-claim:$tool_directory $k8s_config_path gx-short-id + $$uid jobs-directory-claim:$jobs_directory,tool-directory-claim:$tool_directory $k8s_config_path gx-short-id 10 + $$uid jobs-directory-claim:$jobs_directory,tool-directory-claim:$tool_directory $k8s_config_path gx-short-id never + $$uid - 1.9 + 1.1 10M true busybox:ubuntu-14.04 42 - 1.9 + 1.1 10M true busybox:ubuntu-14.04 42 - 1.9 + 1.1 10M true busybox:ubuntu-14.04 @@ -182,7 +186,9 @@ class BaseKubernetesIntegrationTestCase(BaseJobEnvironmentIntegrationTestCase, M super(BaseKubernetesIntegrationTestCase, cls).tearDownClass() @classmethod - def handle_galaxy_config_kwds(cls, config, ): + def handle_galaxy_config_kwds(cls, config): + # TODO: implement metadata setting as separate job, as service or side-car + config['retry_metadata_internally'] = True config["jobs_directory"] = cls.jobs_directory config["file_path"] = cls.jobs_directory config["job_config_file"] = cls.job_config.path diff --git a/test/integration/test_scripts.py b/test/integration/test_scripts.py index cbedbd75f0e..b718f1ec6ca 100644 --- a/test/integration/test_scripts.py +++ b/test/integration/test_scripts.py @@ -116,7 +116,6 @@ class ScriptsIntegrationTestCase(integration_util.IntegrationTestCase): output = self._scripts_check_output(script, ["-c", config_file]) assert "Complete" in output - @integration_util.skip_if_jenkins def test_grt_export(self): script = "grt/export.py" self._scripts_check_argparse_help(script) @@ -134,7 +133,7 @@ class ScriptsIntegrationTestCase(integration_util.IntegrationTestCase): json_file = os.path.join(self.config_dir, json_files[0]) with open(json_file, "r") as f: export = json.load(f) - assert export["version"] == 2 + assert export["version"] == 3 def test_admin_cleanup_datasets(self): self._scripts_check_argparse_help("cleanup_datasets/admin_cleanup_datasets.py") @@ -179,13 +178,21 @@ class ScriptsIntegrationTestCase(integration_util.IntegrationTestCase): clean_env = { "PATH": os.environ.get("PATH", None), } # Don't let testing environment variables interfere with config. - return unicodify(subprocess.check_output(cmd, cwd=cwd, env=clean_env)) + try: + return unicodify(subprocess.check_output(cmd, cwd=cwd, env=clean_env)) + except Exception as e: + if isinstance(e, subprocess.CalledProcessError): + raise Exception("%s\nOutput was:\n%s" % (unicodify(e), e.output)) + raise def write_config_file(self): config_dir = self.config_dir path = os.path.join(config_dir, "galaxy.yml") self._test_driver.temp_directories.extend([config_dir]) + config = self._raw_config + # Update config dict with database_connection, which might be set through env variables + config['database_connection'] = self._app.config.database_connection with open(path, "w") as f: - yaml.dump({"galaxy": self._raw_config}, f) + yaml.dump({"galaxy": config}, f) return path diff --git a/test/unit/jobs/test_job_wrapper.py b/test/unit/jobs/test_job_wrapper.py index 50389f204ef..30289b74118 100644 --- a/test/unit/jobs/test_job_wrapper.py +++ b/test/unit/jobs/test_job_wrapper.py @@ -53,7 +53,7 @@ class BaseWrapperTestCase(UsesApp): def test_version_path(self): wrapper = self._wrapper() version_path = wrapper.get_version_string_path_legacy() - expected_path = os.path.join(self.test_directory, "new_files", "GALAXY_VERSION_STRING_345") + expected_path = os.path.join(self.test_directory, "working", "COMMAND_VERSION") self.assertEqual(version_path, expected_path) def test_prepare_sets_command_line(self): diff --git a/test/unit/test_remote_shell.py b/test/unit/test_remote_shell.py index d460509c3fd..f13305ba2be 100644 --- a/test/unit/test_remote_shell.py +++ b/test/unit/test_remote_shell.py @@ -23,6 +23,7 @@ class TestCliInterface(unittest.TestCase): cls.username = 'testuser' cls.shell_params = {'username': cls.username, 'private_key': cls.ssh_keys.private_key_file, + 'strict_host_key_checking': False, 'hostname': 'localhost'} cls.cli_interface = CliInterface() diff --git a/test/unit/tool_util/test_parsing.py b/test/unit/tool_util/test_parsing.py index da646d19894..27047fe46a2 100644 --- a/test/unit/tool_util/test_parsing.py +++ b/test/unit/tool_util/test_parsing.py @@ -641,6 +641,18 @@ class ExpectationsTestCase(BaseLoaderTestCase): assert len(test_0["stdout"]) == 2 +class ExpectationsCommandVersionTestCase(BaseLoaderTestCase): + source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/job_properties.xml") + source_contents = None + + def test_tests(self): + tests_dict = self._tool_source.parse_tests_to_dict() + tests = tests_dict["tests"] + assert len(tests) > 0 + test_0 = tests[0] + assert len(test_0["command_version"]) == 1 + + class QcStdioTestCase(BaseLoaderTestCase): source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/qc_stdout.xml") source_contents = None diff --git a/test/unit/tools/test_watcher.py b/test/unit/tools/test_watcher.py index 352a4a8a32c..f1e0c12a21b 100644 --- a/test/unit/tools/test_watcher.py +++ b/test/unit/tools/test_watcher.py @@ -4,47 +4,47 @@ from contextlib import contextmanager from os import path from shutil import rmtree +import pytest + from galaxy.tools.toolbox import watcher from galaxy.util import bunch +@pytest.mark.skipif(not watcher.can_watch, reason="watchdog not available") def test_watcher(): - if not watcher.can_watch: - from nose.plugins.skip import SkipTest - raise SkipTest() - with __test_directory() as t: tool_path = path.join(t, "test.xml") toolbox = Toolbox() - open(tool_path, "w").write("a") + with open(tool_path, "w") as f: + f.write("a") tool_watcher = watcher.get_tool_watcher(toolbox, bunch.Bunch( watch_tools=True )) tool_watcher.start() - time.sleep(1) tool_watcher.watch_file(tool_path, "cool_tool") + time.sleep(2) assert not toolbox.was_reloaded("cool_tool") - open(tool_path, "w").write("b") + with open(tool_path, "w") as f: + f.write("b") wait_for_reload(lambda: toolbox.was_reloaded("cool_tool")) tool_watcher.shutdown() assert tool_watcher.observer is None +@pytest.mark.skipif(not watcher.can_watch, reason="watchdog not available") def test_tool_conf_watcher(): - if not watcher.can_watch: - from nose.plugins.skip import SkipTest - raise SkipTest() - callback = CallbackRecorder() conf_watcher = watcher.get_tool_conf_watcher(callback.call) conf_watcher.start() with __test_directory() as t: tool_conf_path = path.join(t, "test_conf.xml") - open(tool_conf_path, "w").write("a") + with open(tool_conf_path, "w") as f: + f.write("a") conf_watcher.watch_file(tool_conf_path) - time.sleep(1) - open(tool_conf_path, "w").write("b") + time.sleep(2) + with open(tool_conf_path, "w") as f: + f.write("b") wait_for_reload(lambda: callback.called) conf_watcher.shutdown() assert conf_watcher.thread is None diff --git a/test/unit/tools/test_wrappers.py b/test/unit/tools/test_wrappers.py index 105f31e0e8e..b57030b805b 100644 --- a/test/unit/tools/test_wrappers.py +++ b/test/unit/tools/test_wrappers.py @@ -109,15 +109,19 @@ def test_raw_object_wrapper(): assert not false_wrapper -def valuewrapper(tool, value, paramtype): +def valuewrapper(tool, value, paramtype, optional=False): if paramtype == "integer": - parameter = IntegerToolParameter(tool, XML('')) + optional = 'optional="true"' if optional else 'value="10"' + parameter = IntegerToolParameter(tool, XML('' % optional)) elif paramtype == "text": - parameter = TextToolParameter(tool, XML('')) + optional = 'optional="true"' if optional else 'value="foo"' + parameter = TextToolParameter(tool, XML('' % optional)) elif paramtype == "float": - parameter = FloatToolParameter(tool, XML('')) + optional = 'optional="true"' if optional else 'value="10.0"' + parameter = FloatToolParameter(tool, XML('' % optional)) elif paramtype == "boolean": - parameter = BooleanToolParameter(tool, XML('')) + optional = 'optional="true"' if optional else 'value=""' + parameter = BooleanToolParameter(tool, XML('' % optional)) return InputValueWrapper(parameter, value) @@ -144,19 +148,24 @@ def test_input_value_wrapper_comparison(tool): @with_mock_tool def test_input_value_wrapper_comparison_optional(tool): - parameter = IntegerToolParameter(tool, XML('')) - wrapper = InputValueWrapper(parameter, None) + wrapper = valuewrapper(tool, None, 'integer', optional=True) assert not wrapper with pytest.raises(ValueError): int(wrapper) assert str(wrapper) == "" assert wrapper == "" # for backward-compatibility - parameter = IntegerToolParameter(tool, XML('')) - wrapper = InputValueWrapper(parameter, 0) + wrapper = valuewrapper(tool, 0, 'integer', optional=True) assert wrapper == 0 assert int(wrapper) == 0 assert str(wrapper) assert wrapper != "" # for backward-compatibility, the correct way to check if an optional integer param is not empty is to use str(wrapper) + wrapper = valuewrapper(tool, None, 'integer', optional=True) + assert wrapper != 1 + assert str(wrapper) == "" + assert wrapper == None # noqa: E711 + wrapper = valuewrapper(tool, None, "boolean") + assert bool(wrapper) is False, wrapper + assert str(wrapper) == 'falsevalue' @with_mock_tool diff --git a/tools/interactive/interactivetool_askomics.xml b/tools/interactive/interactivetool_askomics.xml index cc18516e770..e2ec52f1144 100644 --- a/tools/interactive/interactivetool_askomics.xml +++ b/tools/interactive/interactivetool_askomics.xml @@ -1,7 +1,7 @@ a visual SPARQL query builder - askomics/flaskomics-with-dependencies:3.2.0 + askomics/flaskomics-with-dependencies:3.2.1 @@ -16,6 +16,7 @@ Galaxy ${__user_name__} ${__user_email__} + ${__user_name__} $__galaxy_url__