diff --git a/.k8s_ci.Dockerfile b/.k8s_ci.Dockerfile index 8215825ded4..0129ae9d591 100644 --- a/.k8s_ci.Dockerfile +++ b/.k8s_ci.Dockerfile @@ -22,6 +22,7 @@ ARG STAGE1_BASE=python:3.7-slim ARG FINAL_STAGE_BASE=$STAGE1_BASE ARG GALAXY_USER=galaxy ARG GALAXY_PLAYBOOK_REPO=https://github.com/galaxyproject/galaxy-docker-k8s +ARG GALAXY_PLAYBOOK_BRANCH=v2.0.0 ARG GIT_COMMIT=unspecified ARG BUILD_DATE=unspecified @@ -34,6 +35,7 @@ FROM $STAGE1_BASE AS stage1 ARG DEBIAN_FRONTEND=noninteractive ARG SERVER_DIR ARG GALAXY_PLAYBOOK_REPO +ARG GALAXY_PLAYBOOK_BRANCH # Init Env ENV LC_ALL=en_US.UTF-8 @@ -57,7 +59,7 @@ RUN set -xe; \ WORKDIR /tmp/ansible RUN rm -rf * ENV LC_ALL en_US.UTF-8 -RUN git clone --depth 1 $GALAXY_PLAYBOOK_REPO galaxy-docker +RUN git clone --depth 1 --branch $GALAXY_PLAYBOOK_BRANCH $GALAXY_PLAYBOOK_REPO galaxy-docker WORKDIR /tmp/ansible/galaxy-docker RUN ansible-galaxy install -r requirements.yml -p roles --force-with-deps diff --git a/client/gulpfile.js b/client/gulpfile.js index cd1d656bb1e..05796bd6f16 100644 --- a/client/gulpfile.js +++ b/client/gulpfile.js @@ -130,8 +130,18 @@ function buildPlugins(callback, forceRebuild) { } ); console.log(`Building ${pluginName}`); - child_process.spawnSync("yarn", ["build"], { cwd: f, stdio: "inherit", shell: true }); - child_process.exec(`(git rev-parse HEAD 2>/dev/null || echo \`\`) > ${hashFilePath}`); + + if ( + child_process.spawnSync("yarn", ["build"], { cwd: f, stdio: "inherit", shell: true }).status === + 0 + ) { + console.log(`Successfully built, saving build state to ${hashFilePath}`); + child_process.exec(`(git rev-parse HEAD 2>/dev/null || echo \`\`) > ${hashFilePath}`); + } else { + console.error( + `Error building ${pluginName}, not saving build state. Please report this issue to the Galaxy Team.` + ); + } } }); }); diff --git a/client/src/components/Common/ButtonSpinner.vue b/client/src/components/Common/ButtonSpinner.vue index fe45de35fb9..56531098ef6 100644 --- a/client/src/components/Common/ButtonSpinner.vue +++ b/client/src/components/Common/ButtonSpinner.vue @@ -2,13 +2,7 @@ Please wait... - + {{ title }} @@ -37,10 +31,6 @@ export default { type: String, default: null, }, - disabled: { - type: Boolean, - default: false, - }, }, }; diff --git a/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue b/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue index 6d8543f0e66..4718b9365e1 100644 --- a/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue +++ b/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue @@ -2,18 +2,9 @@
Workflow: {{ model.name }} - +
- - {{ errorMessage }} - - + Expand to full workflow form.
@@ -47,7 +38,6 @@ export default { }, data() { return { - errorMessage: null, formData: {}, inputTypes: {}, }; @@ -80,23 +70,12 @@ export default { }); return inputs; }, - runButtonDisabled() { - return this.errorMessage !== null; - }, }, methods: { - onValidation(validationError) { - this.errorMessage = validationError - ? "Please address the issues highlighted below before proceeding." - : null; - }, onChange(data) { this.formData = data; }, onExecute() { - if (this.errorMessage) { - return; - } const replacementParams = {}; const inputs = {}; for (const inputName in this.formData) { diff --git a/client/src/components/WorkflowInvocationState/WorkflowInvocationState.vue b/client/src/components/WorkflowInvocationState/WorkflowInvocationState.vue index 928e69e9dd4..dc28a0bc158 100644 --- a/client/src/components/WorkflowInvocationState/WorkflowInvocationState.vue +++ b/client/src/components/WorkflowInvocationState/WorkflowInvocationState.vue @@ -133,7 +133,7 @@ export default { return stepStates; }, invocationAndJobTerminal: function () { - return this.invocationSchedulingTerminal && this.jobStatesTerminal; + return !!(this.invocationSchedulingTerminal && this.jobStatesTerminal); }, invocationLink: function () { return getUrl(`workflows/invocations/report?id=${this.invocationId}`); diff --git a/client/src/components/providers/InvocationStepProvider.js b/client/src/components/providers/InvocationStepProvider.js index eca511c3e4e..b2352e4e888 100644 --- a/client/src/components/providers/InvocationStepProvider.js +++ b/client/src/components/providers/InvocationStepProvider.js @@ -3,6 +3,8 @@ import { getAppRoot } from "onload/loadConfig"; import { SingleQueryProvider } from "components/providers/SingleQueryProvider"; import { rethrowSimple } from "utils/simple-error"; +const TERMINAL_JOB_STATES = ["ok", "error", "deleted", "paused"]; + async function getInvocation({ id }) { try { const { data } = await axios.get(`${getAppRoot()}api/invocations/any/steps/${id}`); @@ -12,4 +14,11 @@ async function getInvocation({ id }) { } } -export default SingleQueryProvider(getInvocation); +const stepIsTerminal = (step) => { + const isTerminal = + ["scheduled", "cancelled", "failed"].includes(step.state) && + step.jobs.every((job) => TERMINAL_JOB_STATES.includes(job.state)); + return isTerminal; +}; + +export default SingleQueryProvider(getInvocation, stepIsTerminal); diff --git a/client/src/mvc/ui/ui-misc.js b/client/src/mvc/ui/ui-misc.js index d32c0cdc7da..f6a755145ab 100644 --- a/client/src/mvc/ui/ui-misc.js +++ b/client/src/mvc/ui/ui-misc.js @@ -86,8 +86,7 @@ export var Input = Backbone.View.extend({ input: "_onchange", }, value: function (new_val) { - new_val !== undefined && - this.model.set("value", new_val === null || typeof new_val === "string" ? new_val : ""); + new_val !== undefined && this.model.set("value", typeof new_val === "string" ? new_val : ""); return this.model.get("value"); }, render: function () { diff --git a/config/plugins/visualizations/jqplot/jqplot_bar/src/script.js b/config/plugins/visualizations/jqplot/jqplot_bar/src/script.js index 99dbe854806..bec0b8b510a 100644 --- a/config/plugins/visualizations/jqplot/jqplot_bar/src/script.js +++ b/config/plugins/visualizations/jqplot/jqplot_bar/src/script.js @@ -3,7 +3,7 @@ import _ from "underscore"; import jqplot from 'jqplot-exported/jqplot'; import { LineRenderer } from 'jqplot-exported/LineRenderer'; import { BarRenderer } from 'jqplot-exported/plugins/BarRenderer'; -import { OHLCRenderer } from 'jqplot-exported/plugins/OHLCRenderer'; +import { OHLCRenderer } from 'jqplot-exported/plugins/OhlcRenderer'; import { EnhancedLegendRenderer } from 'jqplot-exported/plugins/EnhancedLegendRenderer'; import { getDomains, makeCategories, makeTickFormat, makeSeries, mapCategories } from "@galaxyproject/charts/lib/utilities/series"; diff --git a/doc/source/_static/22.01-colour.mp4 b/doc/source/_static/22.01-colour.mp4 new file mode 120000 index 00000000000..ac9575e09d3 --- /dev/null +++ b/doc/source/_static/22.01-colour.mp4 @@ -0,0 +1 @@ +../releases/images/22.01-colour.mp4 \ No newline at end of file diff --git a/doc/source/_static/22.01-export.mp4 b/doc/source/_static/22.01-export.mp4 new file mode 120000 index 00000000000..dc52335e958 --- /dev/null +++ b/doc/source/_static/22.01-export.mp4 @@ -0,0 +1 @@ +../releases/images/22.01-export.mp4 \ No newline at end of file diff --git a/doc/source/releases/22.01.rst b/doc/source/releases/22.01.rst new file mode 100644 index 00000000000..f24918cdde9 --- /dev/null +++ b/doc/source/releases/22.01.rst @@ -0,0 +1,957 @@ + +.. to_doc + +22.01 +=============================== + +Enhancements +------------------------------- + +* Swap to Webpack5 + `Pull Request 11710`_ +* Remote / Late evaluation of tools + `Pull Request 12459`_ +* File sources for gdrive, gcs, onedata, basespace + (thanks to `@nuwang `__). + `Pull Request 12500`_ +* Add support for caching the Singularity cached images directory + `Pull Request 12524`_ +* Various extensions and improvements for asserts + (thanks to `@bernt-matthias `__). + `Pull Request 12528`_ +* New interactive tool - iSEE + (thanks to `@neoformit `__). + `Pull Request 12650`_ +* Resumable file uploads with tus.io + `Pull Request 12656`_ +* Make directory path editable in remote files + (thanks to `@OlegZharkov `__). + `Pull Request 12759`_ +* Display tool source in tool form + `Pull Request 12811`_ +* Modify history contents/near endpoint to support new history scroller + `Pull Request 12892`_ +* Vault abstraction for Galaxy + (thanks to `@nuwang `__). + `Pull Request 12940`_ +* Sort icon on List Collection Creator + (thanks to `@assuntad23 `__). + `Pull Request 12945`_ +* Provide more context for lint context + (thanks to `@bernt-matthias `__). + `Pull Request 12978`_ +* Add mulled-hash command to galaxy-tool-util + (thanks to `@simonbray `__). + `Pull Request 13061`_ +* Add parquet to csv datatype converter + `Pull Request 13078`_ +* XML macros: add named yields, tokenized macros, tokens for attributes, and fix replacement of toplevel yield + (thanks to `@bernt-matthias `__). + `Pull Request 13152`_ +* Enable setting SQLALCHEMY_WARN_20 + `Pull Request 13171`_ +* Add new datatype GRIB + (thanks to `@annefou `__). + `Pull Request 13175`_ +* Use Gravity to run Galaxy + `Pull Request 13224`_ +* Add datatype iml_dataset subclassing html + (thanks to `@knutwa-ext `__). + `Pull Request 12099`_ +* Mark sniffer for fastqillumina as not implemented + (thanks to `@bernt-matthias `__). + `Pull Request 12685`_ +* Create sniffer for GRO file type + (thanks to `@simonbray `__). + `Pull Request 12885`_ +* Agp datatype + `Pull Request 12891`_ +* Type updates from @mr-c - part 2 + (thanks to `@common-workflow-lab `__). + `Pull Request 12899`_ +* 3rd round of type updates to support CWL import + (thanks to `@common-workflow-lab `__). + `Pull Request 12913`_ +* Tighten type ignores + (thanks to `@common-workflow-lab `__). + `Pull Request 12965`_ +* Remove is_multi_byte + (thanks to `@bernt-matthias `__). + `Pull Request 12996`_ +* Add ludwig model datatype + (thanks to `@qiagu `__). + `Pull Request 13073`_ +* Add ampvis2 datatype + (thanks to `@bernt-matthias `__). + `Pull Request 13083`_ +* Add Neper and Gmsh datatypes + (thanks to `@jj-umn `__). + `Pull Request 13087`_ +* Add description of inpcrd and prmtop + (thanks to `@thepineapplepirate `__). + `Pull Request 13126`_ +* Add compression converter + (thanks to `@gallardoalba `__). + `Pull Request 13223`_ +* Change display_in_upload to true for PHAR datatype + (thanks to `@simonbray `__). + `Pull Request 13238`_ +* Add gz as datatype + (thanks to `@astrovsky01 `__). + `Pull Request 13271`_ +* Limit required element validation to safe elements + `Pull Request 13299`_ +* Add phyloseq datatype + (thanks to `@bernt-matthias `__). + `Pull Request 13462`_ +* Allow any format in export tool + (thanks to `@astrovsky01 `__). + `Pull Request 13151`_ +* Add bioconductor as valid xref for tools + (thanks to `@bgruening `__). + `Pull Request 13268`_ +* Test cases to verify only admins can import workflows from paths. + `Pull Request 12513`_ +* Remove deprecated workflow run path through the API. + `Pull Request 12516`_ +* Type updates + (thanks to `@common-workflow-lab `__). + `Pull Request 12868`_ +* Allow workflows to automatically filter inputs by tag + (thanks to `@astrovsky01 `__). + `Pull Request 12930`_ +* Enable macros for workflow scheduler config + (thanks to `@bernt-matthias `__). + `Pull Request 13167`_ +* Remove history service worker + `Pull Request 12510`_ +* Implement fastAPI prefix handling, run selenium tests against prefix + `Pull Request 13103`_ +* Hide form fields and button upon successful submission of bug report + `Pull Request 13237`_ +* Fix toolbox cache + `Pull Request 13333`_ +* Limit maximum number of discovered files per job + `Pull Request 13170`_ +* Fixes for HTML sanitization + (thanks to `@abretaud `__). + `Pull Request 13059`_ +* Make admin user list sortable by last login + (thanks to `@cat-bro `__). + `Pull Request 13209`_ +* Support "show all running" in admin job interface + `Pull Request 13215`_ +* Add config variable for setting tus upload directory + `Pull Request 13313`_ +* Document gunicorn and nginx + `Pull Request 13436`_ +* Update interactive tool docs for Gunicorn / dropping uWSGI + `Pull Request 13470`_ +* Add Migrating from uWSGI to Gunicorn and FastAPI document + `Pull Request 13480`_ +* Separated data PVC + (thanks to `@almahmoud `__). + `Pull Request 12345`_ +* Revised download() so parallel get() is utilized. Small refactor. + (thanks to `@kxk302 `__). + `Pull Request 12522`_ +* More linting for requirements + (thanks to `@bernt-matthias `__). + `Pull Request 12527`_ +* Preserve important dataset metadata when importing/exporting histories. + `Pull Request 12537`_ +* Modify description of the "database" label + `Pull Request 12540`_ +* Add Oleg Zharkov to committers group + `Pull Request 12544`_ +* Add David López to committers group + `Pull Request 12545`_ +* Update container_resolvers_conf.xml.sample + (thanks to `@gmauro `__). + `Pull Request 12560`_ +* Add test for dataset HTML sanitization interface + `Pull Request 12573`_ +* FastAPI: migrate history contents routes + (thanks to `@davelopez `__). + `Pull Request 12578`_ +* Update localization directive to trim whitepsace around text content. + `Pull Request 12580`_ +* Added more caching for github actions which use tox and venv + (thanks to `@nuwang `__). + `Pull Request 12632`_ +* More typing for API tests (especially test_workflows). + `Pull Request 12636`_ +* Better model store abstractions. + `Pull Request 12638`_ +* More Python unit test typing fixes. + `Pull Request 12647`_ +* Annotate more tests as requiring Github to be online. + `Pull Request 12649`_ +* Change logged-in-as to use username + (thanks to `@hexylena `__). + `Pull Request 12652`_ +* Add buttons to expand and collapse history dataset display in pages + `Pull Request 12660`_ +* Formalize dependency of galaxy-data package on galaxy.files module. + `Pull Request 12665`_ +* Check types of modules without type annotations in the their signatures on new modules. + `Pull Request 12691`_ +* Rev Python tooling to latest mypy release. + `Pull Request 12695`_ +* Assorted random typing fixes. + `Pull Request 12696`_ +* Add Tag validation in API requests + (thanks to `@davelopez `__). + `Pull Request 12697`_ +* Framework support for integration tests requiring celery. + `Pull Request 12700`_ +* Allow passing Pydantic models as arguments to Celery tasks. + `Pull Request 12701`_ +* Discourage the use of true/falsevalue for boolean test parameters + (thanks to `@bernt-matthias `__). + `Pull Request 12706`_ +* Extend debugging test instructions for GHA + `Pull Request 12707`_ +* Split integration tests more evenly + `Pull Request 12711`_ +* Tool linter: simple test for test params absent from inputs + (thanks to `@bernt-matthias `__). + `Pull Request 12723`_ +* Add test for invoking a workflow on a new autocreated history + (thanks to `@nsoranzo `__). + `Pull Request 12773`_ +* More Python type fixes from deferred data branch. + `Pull Request 12778`_ +* Type fixes for HDAs and LDDAs manager. + `Pull Request 12780`_ +* FastAPI: migrate Dataset Collections API + (thanks to `@davelopez `__). + `Pull Request 12781`_ +* Type fixes for various visualization backend Python files. + `Pull Request 12783`_ +* Add various include/exclude patterns for github workflows + `Pull Request 12787`_ +* Force strict channel priority when building images with mulled + (thanks to `@rhpvorderman `__). + `Pull Request 12788`_ +* More type fixes for various pieces of managers code. + `Pull Request 12801`_ +* Beta history menu reorg, display fixes. + `Pull Request 12802`_ +* More type fixes for managers code. + `Pull Request 12812`_ +* Fixes and test for PDF generation. + `Pull Request 12813`_ +* Upgrade FastAPI to v0.70.0 + (thanks to `@davelopez `__). + `Pull Request 12836`_ +* Stronger typing of upload code. + `Pull Request 12845`_ +* Sync Selenium GitHub workflows + (thanks to `@nsoranzo `__). + `Pull Request 12849`_ +* Update all Python dependencies + (thanks to `@nsoranzo `__). + `Pull Request 12858`_ +* Drop legacy uploadpost + `Pull Request 12861`_ +* Vuefy color input form element + (thanks to `@OlegZharkov `__). + `Pull Request 12867`_ +* Show error codes when running mypy + (thanks to `@nsoranzo `__). + `Pull Request 12894`_ +* Update cwltool pin to 3.1.20211107152837 + (thanks to `@nsoranzo `__). + `Pull Request 12895`_ +* Increase directory check specificity, of particular use with AnVIL + (thanks to `@luke-c-sargent `__). + `Pull Request 12902`_ +* Create codeql-analysis.yml + (thanks to `@common-workflow-lab `__). + `Pull Request 12908`_ +* Improved Upload Provenance and Correctness + `Pull Request 12912`_ +* Update Python dependencies + (thanks to `@galaxybot `__). + `Pull Request 12916`_ +* Lint for option without text or value + (thanks to `@bernt-matthias `__). + `Pull Request 12921`_ +* FastAPI: migrate datasets API + (thanks to `@davelopez `__). + `Pull Request 12924`_ +* Vuefy Number and Slider Form inputs + (thanks to `@OlegZharkov `__). + `Pull Request 12937`_ +* Allow labeling of subworkflow outputs in parent workflow + `Pull Request 12939`_ +* Linting: check for test expecting failure with outputs + (thanks to `@bernt-matthias `__). + `Pull Request 12960`_ +* Add CWL conformance API tests + (thanks to `@nsoranzo `__). + `Pull Request 12966`_ +* Allow unit testing tool init to specify paths. + (thanks to `@common-workflow-lab `__). + `Pull Request 12969`_ +* Lint for inputs with redundant name + (thanks to `@bernt-matthias `__). + `Pull Request 12974`_ +* Support running celery when cwd is not galaxy_root by setting $GALAXY_ROOT_DIR + `Pull Request 12977`_ +* Split `run_tool_raw()` out of `BaseDatasetPopulator.run_tool()` + (thanks to `@nsoranzo `__). + `Pull Request 12994`_ +* Move ``galaxy_tasks`` decorator to ``galaxy.celery`` + `Pull Request 13000`_ +* Show display applications together with visualizations in center panel + `Pull Request 13018`_ +* Update Python dependencies + (thanks to `@galaxybot `__). + `Pull Request 13029`_ +* Explicit singularity cache + (thanks to `@rhpvorderman `__). + `Pull Request 13032`_ +* Add "diff-format" Makefile target + (thanks to `@common-workflow-lab `__). + `Pull Request 13034`_ +* Drop support for Python 3.6, add Python 3.10 + (thanks to `@nsoranzo `__). + `Pull Request 13035`_ +* Allow param_value filter to refer to parameters with multiple values + (thanks to `@bernt-matthias `__). + `Pull Request 13039`_ +* Add support for CWL `CommandLineTool` and `ExpressionTool` to `DynamicToolManager.create_tool()` + (thanks to `@nsoranzo `__). + `Pull Request 13041`_ +* Update Python dependencies + (thanks to `@galaxybot `__). + `Pull Request 13051`_ +* Allow admin access to datasets API + (thanks to `@davelopez `__). + `Pull Request 13060`_ +* Drop non-actionable warnings from LibraryPermissionsWarning + `Pull Request 13063`_ +* Beta History: homogeneous collection indication + (thanks to `@davelopez `__). + `Pull Request 13068`_ +* Add documentation for test/param/metadata in xsd + (thanks to `@bernt-matthias `__). + `Pull Request 13082`_ +* Update Python dependencies + (thanks to `@galaxybot `__). + `Pull Request 13086`_ +* Add tox environment and GitHub workflow for reports startup test + (thanks to `@nsoranzo `__). + `Pull Request 13088`_ +* Allow duplicated static options + (thanks to `@bernt-matthias `__). + `Pull Request 13094`_ +* Beta History: additional small fixes at collection level + (thanks to `@davelopez `__). + `Pull Request 13097`_ +* Add Back button in file source navigation and ftp helper text + (thanks to `@itisAliRH `__). + `Pull Request 13104`_ +* Add job reports for node types and execution times + (thanks to `@gregvonkuster `__). + `Pull Request 13121`_ +* Initial typing for model discover module. + `Pull Request 13123`_ +* Remove Oleg from committers list + `Pull Request 13128`_ +* Remove Dave C. from committers list + `Pull Request 13129`_ +* Update Python dependencies + (thanks to `@galaxybot `__). + `Pull Request 13135`_ +* Fix MetadataValidator + (thanks to `@bernt-matthias `__). + `Pull Request 13139`_ +* Update DatasetError.vue + `Pull Request 13148`_ +* Assorted Data Libraries fixes + (thanks to `@davelopez `__). + `Pull Request 13159`_ +* Provide even more context to the lint context + (thanks to `@bernt-matthias `__). + `Pull Request 13186`_ +* Skip non-Selenium tests if only lib/galaxy_test/selenium is touched + `Pull Request 13227`_ +* Add documentation for output action attributes + (thanks to `@bernt-matthias `__). + `Pull Request 13236`_ +* Add openPBS cli support, rebuild CLI test images + `Pull Request 13251`_ +* Add missing collection types to xsd doc + (thanks to `@bernt-matthias `__). + `Pull Request 13267`_ +* Update dependencies + `Pull Request 13273`_ +* Only create HDAH entries for undeleted HDAs + (thanks to `@bernt-matthias `__). + `Pull Request 13309`_ +* Add version.json for exposing extra api information about docker container + (thanks to `@nuwang `__). + `Pull Request 13433`_ +* Document gunicorn and apache + `Pull Request 13439`_ +* Update jobs and cluster documentation + `Pull Request 13440`_ +* Update performance tracking for Gunicorn + `Pull Request 13441`_ +* Drop interactive environment build instructions from docs + `Pull Request 13442`_ +* Create config package, move app-related functionality to app + `Pull Request 13461`_ +* Pin gravity to 0.10.0 + `Pull Request 13483`_ +* FastAPI: Histories API migration + (thanks to `@davelopez `__). + `Pull Request 12386`_ +* Vue component dangling bracket style fix (prettier upgrade) + `Pull Request 12428`_ +* Remove unused relationships from model + `Pull Request 12468`_ +* Remove workflow API hooks for tool shed. + `Pull Request 12512`_ +* Vueify Form Recursion + `Pull Request 12518`_ +* FastAPI: Add request wrapper for `SessionRequestContext` + (thanks to `@davelopez `__). + `Pull Request 12531`_ +* Update SQLAlchemy to 1.4.25 + `Pull Request 12538`_ +* Establish unittest utils for galaxy-{data,objectstore} packages. + `Pull Request 12546`_ +* Move API services to their own package + (thanks to `@davelopez `__). + `Pull Request 12548`_ +* Establish galaxy-app unit tests. + `Pull Request 12549`_ +* Establish galaxy-job-execution unit test directory. + `Pull Request 12550`_ +* Establish galaxy-job-metrics unit test directory. + `Pull Request 12551`_ +* Fix up small issues around packaging of galaxy-tool-util code. + `Pull Request 12552`_ +* Move more webapps unit tests in test/unit/webapps. + `Pull Request 12554`_ +* Share a tool_util unittest method via packing to fix up unit test dependencies. + `Pull Request 12555`_ +* Better path handling in app unit tests for packaging. + `Pull Request 12556`_ +* Selenium abstractions for dataset information. + `Pull Request 12569`_ +* TS model: declarative mapping + prep for SQLAlchemy 2.0 + `Pull Request 12666`_ +* Remove unused script and ``history_delete`` controller endpoint + `Pull Request 12668`_ +* Remove ui-table backbone component + `Pull Request 12671`_ +* Libraries (legacy) backbone entrypoint and component removal + `Pull Request 12674`_ +* Ignore type in unused function + `Pull Request 12694`_ +* Misc. fixes for SQLAlchemy 2.0 + `Pull Request 12698`_ +* Upgrade to lagom 1.4.1 + (thanks to `@meadsteve `__). + `Pull Request 12703`_ +* Remove `galaxy.util.getfullargspec` + (thanks to `@bernt-matthias `__). + `Pull Request 12755`_ +* Convert install_model to declarative mapping + `Pull Request 12756`_ +* Mapping tests: restructure, refactor, decouple session and engine, improve + `Pull Request 12765`_ +* New pyupgrade cleanups + (thanks to `@nsoranzo `__). + `Pull Request 12774`_ +* Refactor workflow bookmark + (thanks to `@OlegZharkov `__). + `Pull Request 12810`_ +* Replace boolean input parameter with Vue component in forms + `Pull Request 12820`_ +* Replace hidden field input element with Vue + `Pull Request 12841`_ +* Refactor datasets API + (thanks to `@davelopez `__). + `Pull Request 12847`_ +* Vuefy library folder details modal + (thanks to `@OlegZharkov `__). + `Pull Request 12886`_ +* Remove chart utilities and artifacts from code base + `Pull Request 12890`_ +* Vuefy DirectoryUri form input field + (thanks to `@OlegZharkov `__). + `Pull Request 12898`_ +* Simplify Augustus datatype sniffer + (thanks to `@nsoranzo `__). + `Pull Request 12905`_ +* Modify logic in history_contents/near to accommodate scroller + `Pull Request 12914`_ +* FastAPI: Add response wrapper for `SessionRequestContext` + (thanks to `@davelopez `__). + `Pull Request 12929`_ +* Eliminate a lot of while: True loops in datatype code + `Pull Request 12942`_ +* Small tweaks to facilitate deferred dataset implementation. + `Pull Request 12944`_ +* Modifications to next hid_counter generation + `Pull Request 12948`_ +* Don't bind model's MetaData to engine + `Pull Request 12949`_ +* Vuefy library dataset + (thanks to `@OlegZharkov `__). + `Pull Request 12950`_ +* Assorted mapping cleanup + `Pull Request 12951`_ +* Move `wait_for_invocation_and_jobs()` method to `BaseWorkflowPopulator` + (thanks to `@nsoranzo `__). + `Pull Request 12954`_ +* Avoid setting response headers through `trans` in datatypes + (thanks to `@davelopez `__). + `Pull Request 12956`_ +* Use ``lazy="joined"`` instead of ``lazy=False`` + `Pull Request 12962`_ +* Minor type fixes + `Pull Request 12963`_ +* Add type hints to JobHandlerQueue + `Pull Request 12976`_ +* Fix flaky legacy API tests (hopefully) + (thanks to `@davelopez `__). + `Pull Request 12980`_ +* Create select() construct explicitly + `Pull Request 12983`_ +* Remove unused dataset conversion code + `Pull Request 12991`_ +* Remove unnecessary list creations + (thanks to `@nsoranzo `__). + `Pull Request 13002`_ +* Don't use triple-quoted strings as comments + (thanks to `@nsoranzo `__). + `Pull Request 13007`_ +* Misc fixes - round 4 + (thanks to `@nsoranzo `__). + `Pull Request 13026`_ +* Refactor uploadbox / partially remove jQuery + `Pull Request 13047`_ +* Initial refactoring to separate History rendering and provider components + `Pull Request 13049`_ +* Log missing version file as warning. + `Pull Request 13062`_ +* Importing math.inf no longer needs try/except + `Pull Request 13065`_ +* Avoid imports of history provider internals + `Pull Request 13098`_ +* Drop tool migration scripts and related code + `Pull Request 13099`_ +* Remove duplicated pydantic models + (thanks to `@davelopez `__). + `Pull Request 13125`_ +* Remove migrate tools-related code + `Pull Request 13163`_ +* Replace backref with back_populates + `Pull Request 13165`_ +* Prevent a future regression + `Pull Request 13218`_ +* Remove `jquery.complexify` from everywhere + (thanks to `@itisAliRH `__). + `Pull Request 13228`_ +* Replace custom tour ids in forms with regular DOM identifiers + `Pull Request 13252`_ +* Drop pkg_resources + `Pull Request 13454`_ + + +Fixes +------------------------------- + +* Large peptide IDs + (thanks to `@reid-wagner `__). + `Pull Request 13261`_ +* Try to unit test more of DictParser in Viz component. + `Pull Request 13385`_ +* Allow trailing semicolon in last column of gff3 + (thanks to `@bernt-matthias `__). + `Pull Request 12673`_ +* Fix gff-version header sniffing + (thanks to `@nsoranzo `__). + `Pull Request 12676`_ +* Fix gff and scidx parsing + (thanks to `@bernt-matthias `__). + `Pull Request 12687`_ +* Version bump galaxy tool util and converter dependency fix + `Pull Request 12844`_ +* Downgrade bowtie requirement for CONVERTER_fasta_to_bowtie_color_index + (thanks to `@nsoranzo `__). + `Pull Request 12851`_ +* Fix constructive_solid_geometry.py for non_utf8_error + (thanks to `@jj-umn `__). + `Pull Request 13057`_ +* Show yaml files in preview + `Pull Request 13176`_ +* Enable display of BAM data without sequence info + (thanks to `@wm75 `__). + `Pull Request 13219`_ +* Fix containerized NCBI Datasets Genomes data source tool + `Pull Request 12915`_ +* Update interactivetool_pyiron.xml + (thanks to `@gmauro `__). + `Pull Request 12990`_ +* Add dynamic step to FormNumber range slider + (thanks to `@davelopez `__). + `Pull Request 13233`_ +* FormNumber: notify value change on slider change + (thanks to `@davelopez `__). + `Pull Request 13260`_ +* Always display digest in name for refgenie assets. + `Pull Request 13403`_ +* Fix `UnboundLocalError` in sort collection tool + (thanks to `@davelopez `__). + `Pull Request 13429`_ +* Misc fixes - round 2 + (thanks to `@nsoranzo `__). + `Pull Request 12982`_ +* Fix workflow connections for outputs that are not displayed in uploads + `Pull Request 13174`_ +* Sync tool identifier string with tool version in workflow editor + `Pull Request 13185`_ +* Allow mapping of collection outputs to parameter inputs + `Pull Request 13189`_ +* Fix tool id update when updating tool shed tools in editor + `Pull Request 13196`_ +* Ensure tool_id/content_id is updated when refactoring workflows + (thanks to `@simonbray `__). + `Pull Request 13231`_ +* Improve null value handling + `Pull Request 13285`_ +* Fix construction of SelectToolParameter default value + `Pull Request 13293`_ +* RxJS major version upgrade, client unit testing enhancements. + `Pull Request 12686`_ +* Polyfill overhaul, fixes. + `Pull Request 12889`_ +* Fix publicPath and asset handling when using webpack-dev-server for HMR + `Pull Request 13155`_ +* Bump markdown-it from 12.2.0 to 12.3.2 in /client + `Pull Request 13205`_ +* Bump engine.io from 4.1.1 to 4.1.2 in /client + `Pull Request 13206`_ +* Bump follow-redirects from 1.14.4 to 1.14.7 in /client + `Pull Request 13207`_ +* Bump log4js from 6.3.0 to 6.4.0 in /client + `Pull Request 13225`_ +* Bump nanoid from 3.1.29 to 3.2.0 in /client + `Pull Request 13226`_ +* Guard against instance of GenomeIndexToolData + `Pull Request 12523`_ +* Fix job runner shutdown + `Pull Request 12744`_ +* Fix parsing of yml job conf file in galaxy.dependencies + (thanks to `@cat-bro `__). + `Pull Request 12893`_ +* Rev pulsar-galaxy-lib. + `Pull Request 12896`_ +* Don't pass all Galaxy environment variables to jobs + (thanks to `@nsoranzo `__). + `Pull Request 13011`_ +* Unit test fixes + (thanks to `@nsoranzo `__). + `Pull Request 13033`_ +* Fix re-run with rerun_remap_job_id hiding collection + `Pull Request 13157`_ +* Fix univa job runner setup without nativeSpecification + (thanks to `@Smithsonian `__). + `Pull Request 13301`_ +* Make __link_file_check more robust + `Pull Request 13316`_ +* Don't record version command as part of command line + `Pull Request 13317`_ +* Fix resuming job when job has optional data parameters + `Pull Request 13325`_ +* Allow UNIVA destinations without runtime and memory max given in the nativespec + (thanks to `@bernt-matthias `__). + `Pull Request 13343`_ +* Fix header content-length, add test for send_file + `Pull Request 13120`_ +* Add option for disabling tool shed datatypes... + `Pull Request 13250`_ +* Drop OpenID post authentication protocols + `Pull Request 13281`_ +* Restore Metadata size limit + `Pull Request 13315`_ +* Drop uWSGI from scaling documentation + `Pull Request 13381`_ +* Reimplement collection order test, add wrong order, add `expect_test_failure` attribute + (thanks to `@bernt-matthias `__). + `Pull Request 10304`_ +* Fix test_sharing_valid_by_id selenium test + (thanks to `@OlegZharkov `__). + `Pull Request 12547`_ +* Update and correct sample object store config file + `Pull Request 12571`_ +* Fix ``user_library_import_symlink_allowlist`` option + (thanks to `@scholtalbers `__). + `Pull Request 12608`_ +* Fix selenium populators to respect the json parameter. + `Pull Request 12626`_ +* Fix how beta history panel decides when downloads are available. + `Pull Request 12642`_ +* Typing fixes for various unit tests. + `Pull Request 12644`_ +* Fix database display for beta history panel. + `Pull Request 12646`_ +* Fix deprecation warnings from the `threading` module + (thanks to `@nsoranzo `__). + `Pull Request 12659`_ +* Join thread that hosts asyncio loop for embedded uvicorn + `Pull Request 12710`_ +* Removing unusable Name field from Collections upload + (thanks to `@assuntad23 `__). + `Pull Request 12736`_ +* Removing last character of pair name for Paired List Collections if it's a . or _ + (thanks to `@assuntad23 `__). + `Pull Request 12739`_ +* Avoid double '/' when generating fully qualified API URLs in API tests + (thanks to `@davelopez `__). + `Pull Request 12791`_ +* Create new history for anonymous API users via history/current_histor… + `Pull Request 12829`_ +* Avoid console error when cloning newly created nodes + `Pull Request 12831`_ +* GxITs: don't always add trailing slash to entry point url + (thanks to `@abretaud `__). + `Pull Request 12833`_ +* fix typo in bookmarking + (thanks to `@OlegZharkov `__). + `Pull Request 12838`_ +* Fix bookmarking workflow twice + (thanks to `@OlegZharkov `__). + `Pull Request 12840`_ +* Temporary downgrade FastAPI to v0.68.2 + (thanks to `@davelopez `__). + `Pull Request 12856`_ +* Fix startup with edam + (thanks to `@abretaud `__). + `Pull Request 12860`_ +* Fixes for packages + (thanks to `@nsoranzo `__). + `Pull Request 12883`_ +* Fix `parse_input_sources()` method to return a list + (thanks to `@nsoranzo `__). + `Pull Request 12888`_ +* Consider a history not new once there are any items in it + `Pull Request 12897`_ +* Bump tensorflow from 2.5.1 to 2.5.2 in /lib/galaxy/dependencies + `Pull Request 12900`_ +* Sync package deps with `pyproject.toml` + (thanks to `@nsoranzo `__). + `Pull Request 12904`_ +* Fix menu overflow in right panel + `Pull Request 12906`_ +* Tweak docs for container_resolver_* options + `Pull Request 12910`_ +* Fix installation of tools with biotools id + `Pull Request 12919`_ +* Fix linter for select inputs + (thanks to `@bernt-matthias `__). + `Pull Request 12920`_ +* Fix OpenAPI docs not rendering + (thanks to `@davelopez `__). + `Pull Request 12923`_ +* Fix default value of `runners` in `job_conf_dict` + (thanks to `@nsoranzo `__). + `Pull Request 12926`_ +* Provide 2 types of scoped_session in DI (initially "type of session parameter should be scoped_session") + `Pull Request 12931`_ +* Move the `titleonly` option to input element component + `Pull Request 12932`_ +* fix value_in_data_table validator for multiple selects + (thanks to `@bernt-matthias `__). + `Pull Request 12936`_ +* Update link to "Python for Biologists" page. + (thanks to `@neoformit `__). + `Pull Request 12946`_ +* Try cimg/node:14 to fix circleci js builds + `Pull Request 12947`_ +* Misc fixes + (thanks to `@nsoranzo `__). + `Pull Request 12972`_ +* BugFix: Update social-auth-core library version to fix OIDC problem + (thanks to `@IslamHeggy `__). + `Pull Request 12973`_ +* Fix Rerun in the new form + (thanks to `@OlegZharkov `__). + `Pull Request 12984`_ +* Show error message when history id is invalid + (thanks to `@OlegZharkov `__). + `Pull Request 12985`_ +* Reconstruct separate scoped session for ToolShedRepositoryCache + `Pull Request 12989`_ +* Misc fixes - round 3 + (thanks to `@nsoranzo `__). + `Pull Request 12993`_ +* Bump python-ldap from 3.2.0 to 3.4.0 in /lib/galaxy/dependencies + `Pull Request 13003`_ +* Fix known marcos typo + `Pull Request 13009`_ +* Refactor Sentry middleware/client changes for Tool Shed + `Pull Request 13010`_ +* Drop stream_template_mako function, replace with fill_template_mako + `Pull Request 13017`_ +* Copy HDCA member items to new history when copying history + `Pull Request 13021`_ +* Make dataset display route sessionless as well as anonymous + `Pull Request 13024`_ +* Trigger visualization update on chart initialization + `Pull Request 13025`_ +* Fix running test/integration/test_remote_files.py directly. + `Pull Request 13038`_ +* Misc fixes - round 5 + (thanks to `@nsoranzo `__). + `Pull Request 13044`_ +* Fix dynamic optional + multiple select parameters: allow to select no option + (thanks to `@bernt-matthias `__). + `Pull Request 13048`_ +* Add tools to IMPLICITLY_REQUIRED_TOOL_FILES in tools/__init__.py + (thanks to `@cat-bro `__). + `Pull Request 13050`_ +* Only check profile version in Galaxy + `Pull Request 13055`_ +* Fix private history publishing + (thanks to `@davelopez `__). + `Pull Request 13056`_ +* Improve dataset search help + (thanks to `@wm75 `__). + `Pull Request 13058`_ +* Change default `file_path` variable for galaxy reports + (thanks to `@quacksawbones `__). + `Pull Request 13067`_ +* Drop old sqlalchemy patterns that break reports app + `Pull Request 13069`_ +* Add iRODS resource name in calls to irods + (thanks to `@kxk302 `__). + `Pull Request 13072`_ +* Fix Cloud object store + (thanks to `@bioflowy `__). + `Pull Request 13074`_ +* Use yield_per option to limit amount of ORM objects loaded into memory + `Pull Request 13076`_ +* Fix issue with uploading cached file to irods + (thanks to `@kxk302 `__). + `Pull Request 13080`_ +* Fix tag attribute parsing on input element + (thanks to `@astrovsky01 `__). + `Pull Request 13081`_ +* Fix dataset details button when using prefix + `Pull Request 13092`_ +* Fix center panel reload when clicking help button + `Pull Request 13093`_ +* Limit possible size of info field, read only 100 MB of stdio + `Pull Request 13096`_ +* Disable cleanup thread for cloud object store in set metadata + (thanks to `@bioflowy `__). + `Pull Request 13102`_ +* Prevent select all in FilesDialog when no entries and select empty resource + (thanks to `@itisAliRH `__). + `Pull Request 13107`_ +* Fix selenium test via dropping redundant test case + `Pull Request 13112`_ +* Delay iframe creation for news until clicked + `Pull Request 13116`_ +* Webhooks robustification for news and gtn + `Pull Request 13118`_ +* Fix routing for tool execution from gtn webhook + `Pull Request 13130`_ +* Fix history import for datasets with tags + `Pull Request 13134`_ +* Add more required dependencies for installed Galaxy + `Pull Request 13143`_ +* Path resolution fixes for installed Galaxy + `Pull Request 13147`_ +* Limit lifetime of repository cache + `Pull Request 13154`_ +* converting forward and reverse filters to reactive Vue array + (thanks to `@assuntad23 `__). + `Pull Request 13166`_ +* Fix stdio parsing + (thanks to `@bernt-matthias `__). + `Pull Request 13178`_ +* Catch exceptions when job.user is None + (thanks to `@rhpvorderman `__). + `Pull Request 13179`_ +* Fix redirect to login page when login is required to perform operations + `Pull Request 13188`_ +* Fix workflow invocation name overflow + (thanks to `@davelopez `__). + `Pull Request 13193`_ +* Fix metadata source access in tool shed + `Pull Request 13202`_ +* Fix error getting toolshed repo metadata when no repo found + (thanks to `@simonbray `__). + `Pull Request 13203`_ +* Bugfix for allowing +'s in tool urls + `Pull Request 13204`_ +* Fix test_deserialize_* unit tests + (thanks to `@nsoranzo `__). + `Pull Request 13208`_ +* Fix hide collections in batch + (thanks to `@davelopez `__). + `Pull Request 13214`_ +* Handle unique tag name constraints + `Pull Request 13216`_ +* Ensure that jQuery updates drop target locations during dragging + `Pull Request 13217`_ +* Fix adding new files via tarball upload + `Pull Request 13230`_ +* Initialize error attribute in input model for workflows + `Pull Request 13246`_ +* Fix imported sub-workflows cannot be edited + (thanks to `@davelopez `__). + `Pull Request 13254`_ +* Fix expect_test_failure for expect_num_outputs + (thanks to `@bernt-matthias `__). + `Pull Request 13265`_ +* Fix gravity requirement for Poetry + (thanks to `@nsoranzo `__). + `Pull Request 13278`_ +* Fix dataset details and drag and drop + `Pull Request 13290`_ +* Fix stopping ITs on slurm + `Pull Request 13304`_ +* Ensure Notification API is available prior to trying to use it + `Pull Request 13322`_ +* Fix toolbox handling of spaces in old tools + `Pull Request 13347`_ +* Fix macro parsing + (thanks to `@bernt-matthias `__). + `Pull Request 13373`_ +* Clarify that history exports are not kept forever + `Pull Request 13389`_ +* Fix history search help popover not closing on click outside + (thanks to `@davelopez `__). + `Pull Request 13390`_ +* Ensure $GRAVITY_STATE_DIR is present in venv activate script any time run.sh/common_startup.sh is called normally + `Pull Request 13392`_ +* Use a2wsgi to serve WSGI app + `Pull Request 13396`_ +* Dataset annotations are optional and should not be required + `Pull Request 13420`_ +* Fix repository uploads with remove_repo_files_not_in_tar + `Pull Request 13425`_ +* Wrap reports app in static, fix tool shed entry point + `Pull Request 13430`_ +* Initial fix for quay.io repo query issue + `Pull Request 13468`_ +* Add profile version to ethercalc + `Pull Request 13469`_ +* Backport and fix webapps tests + `Pull Request 13478`_ +* Use timeout instead of interval when polling, optionally terminate polling + `Pull Request 13485`_ + +.. include:: 22.01_prs.rst + diff --git a/doc/source/releases/22.01_announce.rst b/doc/source/releases/22.01_announce.rst new file mode 100644 index 00000000000..26dd3a9621d --- /dev/null +++ b/doc/source/releases/22.01_announce.rst @@ -0,0 +1,125 @@ + +=========================================================== +January 2022 Galaxy Release (v 22.01) +=========================================================== + +.. include:: _header.rst + +Highlights +=========================================================== + +**Galaxy starts as FastAPI application by default** + Starting Galaxy via ``run.sh`` will use the new `gravity process manager + `__. + The new configuration uses `gunicorn `__ and `FastAPI `__ to drive Galaxy's web process and starts job handler and `Celery `__ processes automatically. + For more details and instructions please consult the `Migrating to Gunicorn documentation `__. + (`Pull Request 13224`_). + +**User Preferences can be encrypted in Galaxy Vault** + Galaxy can now be configured to store secrets in an external vault, which is useful for secure handling and centralization of secrets management. + In particular, information fields in the "Manage information" section of the user profile, such as dropbox keys, can be configured to be encrypted + at rest in a vault (Hashicorp, Custos or database) instead of being stored as plain text in the user preferences table. For detailed information on + configuration, refer to the `vault section `__ of the admin documentation. + (`Pull Request 12940`_). + +**Resumable uploads via tus.io, improved upload provenance** + Galaxy now supports chunked and resumable uploads via the `tus protocol `__. + This is more performant and easier to setup than the current nginx-based + solutions. For detailed configuration instructions, see the `documentation `__. + Galaxy now displays all manipulations that were performed during an upload, + such as newline conversions, converting spaces to tabs or dataset grooming. + (`Pull Request 12656`_, `Pull Request 12861`_, `Pull Request 13313`_, `Pull Request 12912`_). + +Also check out the `22.01 user release notes <22.01_announce_user.html>`__ + +Get Galaxy +========== + +The code lives at `GitHub `__ and you should have `Git `__ to obtain it. + +To get a new Galaxy repository run: + .. code-block:: shell + + $ git clone -b release_22.01 https://github.com/galaxyproject/galaxy.git + +To update an existing Galaxy repository run: + .. code-block:: shell + + $ git fetch origin && git checkout release_22.01 && git pull --ff-only origin release_22.01 + +See the `community hub `__ for additional details on source code locations. + +Deprecation Notices +=========================================================== +PostgreSQL 9.x is now EOL and is no longer supported. Please upgrade to PostgreSQL 10 or higher. +Python 3.6 is officially EOL and Galaxy no longer tests or maintains compatibility. +This release will be the last Galaxy release that can run under Paste or uWSGI. +Please consult the `Migrating to Gunicorn documentation `__ for . +instructions on how to upgrade. + +Configuration Changes +===================== + +Added +----- + +The following configuration options are new + +config/galaxy.yml.sample:galaxy +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- mapping.galaxy.mapping.enable_tool_source_display +- mapping.galaxy.mapping.galaxy_url_prefix +- mapping.galaxy.mapping.load_tool_shed_datatypes +- mapping.galaxy.mapping.max_discovered_files +- mapping.galaxy.mapping.maximum_upload_file_size +- mapping.galaxy.mapping.tool_evaluation_strategy +- mapping.galaxy.mapping.tus_upload_store +- mapping.galaxy.mapping.vault_config_file + + +Changed +------- + +The following configuration options have been changed + +config/galaxy.yml.sample:galaxy +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- mapping.reports.mapping.file_path.default has changed from + + :: + + database/files + + to + + :: + + database/objects + + + + +Removed +------- + +The following configuration options have been completely removed + +config/galaxy.yml.sample:galaxy +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- mapping.galaxy.mapping.check_migrate_tools +- mapping.galaxy.mapping.enable_openid +- mapping.galaxy.mapping.force_beta_workflow_scheduled_for_collections +- mapping.galaxy.mapping.force_beta_workflow_scheduled_min_steps +- mapping.galaxy.mapping.openid_consumer_cache_path + + +Release Notes +=========================================================== + +.. include:: 22.01.rst + :start-after: announce_start + +.. include:: _thanks.rst diff --git a/doc/source/releases/22.01_announce_user.rst b/doc/source/releases/22.01_announce_user.rst new file mode 100644 index 00000000000..255ddeb0ba9 --- /dev/null +++ b/doc/source/releases/22.01_announce_user.rst @@ -0,0 +1,159 @@ + +=========================================================== +January 2022 Galaxy Release (v 22.01) +=========================================================== + +.. include:: _header.rst + +Highlights +=========================================================== + +**New Colour Selector** + +Do you use tools which require colour inputs, like Circos? Previously we had a restricted colour input which gave you a very limited palette. Now, you have complete freedom of choice with the modern colour selector. + +.. raw:: html + + + + +**Improved File Export** + +If you've been exporting files from Galaxy lately, you've probably seen the amazing new remote file source export, which allows you to export files to FTP, Dropbox, and other locations. + +.. raw:: html + + + +**Improved File Uploads** + +Previously, Galaxy servers used a variety of methods to let you upload large files easily, including some servers which required FTP for large files. We have replaced this with a new upload method, which will be enabled on all usegalaxy servers soon. This will make file uploads significantly smoother, and will be more tolerant of network failures and interruptions! You do not need to make any changes. See `Pull Request 12656`_ for more information. + +**Beta History: Collection Improvements** + +If you've been trying out the beta history (which will be the default history next release!), it has been updated to indicate whether collections are homogeneous or heterogeneous. This will help you see more easily if you've accidentally included an incorrect dataset. Test out the beta history today and let us know if you have any issues! + +.. figure:: https://user-images.githubusercontent.com/46503462/146426341-d16d07d8-164b-40ef-976a-52e73d94bfc9.png + :alt: Screenshot of a history named collection tests history using the new beta history panel. Several collections are shown below, some labelled "A list of 6 txt datasets" and some labelled "a list of 3 datasets", without a datatype shown. + + +New Visualizations +=========================================================== + +.. visualizations + +Thanks to `@neoformit `__, a new R shiny app "iSEE" has been added, which provides a UI for visualization of single-cell RNA datasets. See `Pull Request 12650`_ for more information + +.. figure:: https://user-images.githubusercontent.com/42562517/136135157-40a46a31-3553-49b3-a975-b518db1374b2.png + :alt: screenshot of the iSEE app, on the left is a colourful scatterplot with dots grouped, labelled reduced dimension plot. On the right is a plot labelled feature assay plot with a number of violin plots. Both plots feature controls at the bottom for modifying the visualisation's parameters. + +New Datatypes +=========================================================== + +.. datatypes +* Add datatype iml_dataset subclassing html + (thanks to `@knutwa-ext `__). + `Pull Request 12099`_ +* Create sniffer for GRO file type + (thanks to `@simonbray `__). + `Pull Request 12885`_ +* Agp datatype + `Pull Request 12891`_ +* Add ludwig model datatype + (thanks to `@qiagu `__). + `Pull Request 13073`_ +* Add ampvis2 datatype + (thanks to `@bernt-matthias `__). + `Pull Request 13083`_ +* Add Neper and Gmsh datatypes + (thanks to `@jj-umn `__). + `Pull Request 13087`_ +* Add new datatype GRIB + (thanks to `@annefou `__). + `Pull Request 13175`_ +* Show yaml files in preview + `Pull Request 13176`_ +* Enable display of BAM data without sequence info + (thanks to `@wm75 `__). + `Pull Request 13219`_ +* Add gz as datatype + (thanks to `@astrovsky01 `__). + `Pull Request 13271`_ +* Add phyloseq datatype + (thanks to `@bernt-matthias `__). + `Pull Request 13462`_ + +Builtin Tool Updates +=========================================================== + +.. tools +* New interactive tool - iSEE + (thanks to `@neoformit `__). + `Pull Request 12650`_ +* Ignore type in unused function + `Pull Request 12694`_ +* Merge 21.09 -> dev + `Pull Request 12747`_ +* Set profile version of ncbi_datasets so it can run containerized + `Pull Request 12819`_ +* Downgrade bowtie requirement for CONVERTER_fasta_to_bowtie_color_index + (thanks to `@nsoranzo `__). + `Pull Request 12851`_ +* Fix containerized NCBI Datasets Genomes data source tool + `Pull Request 12915`_ +* Tighten type ignores + (thanks to `@common-workflow-lab `__). + `Pull Request 12965`_ +* Update interactivetool_pyiron.xml + (thanks to `@gmauro `__). + `Pull Request 12990`_ +* Don't use triple-quoted strings as comments + (thanks to `@nsoranzo `__). + `Pull Request 13007`_ +* Add parquet to csv datatype converter + `Pull Request 13078`_ +* Drop tool migration scripts and related code + `Pull Request 13099`_ +* Allow any format in export tool + (thanks to `@astrovsky01 `__). + `Pull Request 13151`_ +* Add dynamic step to FormNumber range slider + (thanks to `@davelopez `__). + `Pull Request 13233`_ +* FormNumber: notify value change on slider change + (thanks to `@davelopez `__). + `Pull Request 13260`_ +* Add bioconductor as valid xref for tools + (thanks to `@bgruening `__). + `Pull Request 13268`_ + +Release Testing Team +=========================================================== + +A special thanks to the release testing team for testing many of the new features and reporting many bugs: + +- `Keith Suderman `_ +- `Delphine Lariviere `_ +- `Jennifer Hillman-Jackson `_ +- `Natalie Kucher `_ +- `Jennifer Hillman-Jackson `_ +- `Alex Ostrovsky `_ +- `Enis Afgan `_ +- `Jennifer Hillman-Jackson `_ +- `Alex Ostrovsky `_ +- `Enis Afgan `_ + +Release Notes +=========================================================== + +User facing release notes compiled by `Helena Rasche `_. + +Please see the `full release notes <22.01_announce.html>`_ for more details. + +.. include:: 22.01_prs.rst + +.. include:: _thanks.rst diff --git a/doc/source/releases/22.01_prs.rst b/doc/source/releases/22.01_prs.rst new file mode 100644 index 00000000000..81033ac826f --- /dev/null +++ b/doc/source/releases/22.01_prs.rst @@ -0,0 +1,417 @@ + +.. github_links +.. _Pull Request 10304: https://github.com/galaxyproject/galaxy/pull/10304 +.. _Pull Request 11710: https://github.com/galaxyproject/galaxy/pull/11710 +.. _Pull Request 12099: https://github.com/galaxyproject/galaxy/pull/12099 +.. _Pull Request 12345: https://github.com/galaxyproject/galaxy/pull/12345 +.. _Pull Request 12386: https://github.com/galaxyproject/galaxy/pull/12386 +.. _Pull Request 12428: https://github.com/galaxyproject/galaxy/pull/12428 +.. _Pull Request 12459: https://github.com/galaxyproject/galaxy/pull/12459 +.. _Pull Request 12468: https://github.com/galaxyproject/galaxy/pull/12468 +.. _Pull Request 12500: https://github.com/galaxyproject/galaxy/pull/12500 +.. _Pull Request 12510: https://github.com/galaxyproject/galaxy/pull/12510 +.. _Pull Request 12512: https://github.com/galaxyproject/galaxy/pull/12512 +.. _Pull Request 12513: https://github.com/galaxyproject/galaxy/pull/12513 +.. _Pull Request 12516: https://github.com/galaxyproject/galaxy/pull/12516 +.. _Pull Request 12518: https://github.com/galaxyproject/galaxy/pull/12518 +.. _Pull Request 12522: https://github.com/galaxyproject/galaxy/pull/12522 +.. _Pull Request 12523: https://github.com/galaxyproject/galaxy/pull/12523 +.. _Pull Request 12524: https://github.com/galaxyproject/galaxy/pull/12524 +.. _Pull Request 12527: https://github.com/galaxyproject/galaxy/pull/12527 +.. _Pull Request 12528: https://github.com/galaxyproject/galaxy/pull/12528 +.. _Pull Request 12531: https://github.com/galaxyproject/galaxy/pull/12531 +.. _Pull Request 12537: https://github.com/galaxyproject/galaxy/pull/12537 +.. _Pull Request 12538: https://github.com/galaxyproject/galaxy/pull/12538 +.. _Pull Request 12540: https://github.com/galaxyproject/galaxy/pull/12540 +.. _Pull Request 12544: https://github.com/galaxyproject/galaxy/pull/12544 +.. _Pull Request 12545: https://github.com/galaxyproject/galaxy/pull/12545 +.. _Pull Request 12546: https://github.com/galaxyproject/galaxy/pull/12546 +.. _Pull Request 12547: https://github.com/galaxyproject/galaxy/pull/12547 +.. _Pull Request 12548: https://github.com/galaxyproject/galaxy/pull/12548 +.. _Pull Request 12549: https://github.com/galaxyproject/galaxy/pull/12549 +.. _Pull Request 12550: https://github.com/galaxyproject/galaxy/pull/12550 +.. _Pull Request 12551: https://github.com/galaxyproject/galaxy/pull/12551 +.. _Pull Request 12552: https://github.com/galaxyproject/galaxy/pull/12552 +.. _Pull Request 12554: https://github.com/galaxyproject/galaxy/pull/12554 +.. _Pull Request 12555: https://github.com/galaxyproject/galaxy/pull/12555 +.. _Pull Request 12556: https://github.com/galaxyproject/galaxy/pull/12556 +.. _Pull Request 12560: https://github.com/galaxyproject/galaxy/pull/12560 +.. _Pull Request 12569: https://github.com/galaxyproject/galaxy/pull/12569 +.. _Pull Request 12571: https://github.com/galaxyproject/galaxy/pull/12571 +.. _Pull Request 12573: https://github.com/galaxyproject/galaxy/pull/12573 +.. _Pull Request 12578: https://github.com/galaxyproject/galaxy/pull/12578 +.. _Pull Request 12580: https://github.com/galaxyproject/galaxy/pull/12580 +.. _Pull Request 12608: https://github.com/galaxyproject/galaxy/pull/12608 +.. _Pull Request 12626: https://github.com/galaxyproject/galaxy/pull/12626 +.. _Pull Request 12632: https://github.com/galaxyproject/galaxy/pull/12632 +.. _Pull Request 12636: https://github.com/galaxyproject/galaxy/pull/12636 +.. _Pull Request 12638: https://github.com/galaxyproject/galaxy/pull/12638 +.. _Pull Request 12642: https://github.com/galaxyproject/galaxy/pull/12642 +.. _Pull Request 12644: https://github.com/galaxyproject/galaxy/pull/12644 +.. _Pull Request 12646: https://github.com/galaxyproject/galaxy/pull/12646 +.. _Pull Request 12647: https://github.com/galaxyproject/galaxy/pull/12647 +.. _Pull Request 12649: https://github.com/galaxyproject/galaxy/pull/12649 +.. _Pull Request 12650: https://github.com/galaxyproject/galaxy/pull/12650 +.. _Pull Request 12652: https://github.com/galaxyproject/galaxy/pull/12652 +.. _Pull Request 12656: https://github.com/galaxyproject/galaxy/pull/12656 +.. _Pull Request 12659: https://github.com/galaxyproject/galaxy/pull/12659 +.. _Pull Request 12660: https://github.com/galaxyproject/galaxy/pull/12660 +.. _Pull Request 12664: https://github.com/galaxyproject/galaxy/pull/12664 +.. _Pull Request 12665: https://github.com/galaxyproject/galaxy/pull/12665 +.. _Pull Request 12666: https://github.com/galaxyproject/galaxy/pull/12666 +.. _Pull Request 12668: https://github.com/galaxyproject/galaxy/pull/12668 +.. _Pull Request 12671: https://github.com/galaxyproject/galaxy/pull/12671 +.. _Pull Request 12673: https://github.com/galaxyproject/galaxy/pull/12673 +.. _Pull Request 12674: https://github.com/galaxyproject/galaxy/pull/12674 +.. _Pull Request 12676: https://github.com/galaxyproject/galaxy/pull/12676 +.. _Pull Request 12685: https://github.com/galaxyproject/galaxy/pull/12685 +.. _Pull Request 12686: https://github.com/galaxyproject/galaxy/pull/12686 +.. _Pull Request 12687: https://github.com/galaxyproject/galaxy/pull/12687 +.. _Pull Request 12691: https://github.com/galaxyproject/galaxy/pull/12691 +.. _Pull Request 12694: https://github.com/galaxyproject/galaxy/pull/12694 +.. _Pull Request 12695: https://github.com/galaxyproject/galaxy/pull/12695 +.. _Pull Request 12696: https://github.com/galaxyproject/galaxy/pull/12696 +.. _Pull Request 12697: https://github.com/galaxyproject/galaxy/pull/12697 +.. _Pull Request 12698: https://github.com/galaxyproject/galaxy/pull/12698 +.. _Pull Request 12700: https://github.com/galaxyproject/galaxy/pull/12700 +.. _Pull Request 12701: https://github.com/galaxyproject/galaxy/pull/12701 +.. _Pull Request 12703: https://github.com/galaxyproject/galaxy/pull/12703 +.. _Pull Request 12706: https://github.com/galaxyproject/galaxy/pull/12706 +.. _Pull Request 12707: https://github.com/galaxyproject/galaxy/pull/12707 +.. _Pull Request 12710: https://github.com/galaxyproject/galaxy/pull/12710 +.. _Pull Request 12711: https://github.com/galaxyproject/galaxy/pull/12711 +.. _Pull Request 12723: https://github.com/galaxyproject/galaxy/pull/12723 +.. _Pull Request 12736: https://github.com/galaxyproject/galaxy/pull/12736 +.. _Pull Request 12739: https://github.com/galaxyproject/galaxy/pull/12739 +.. _Pull Request 12744: https://github.com/galaxyproject/galaxy/pull/12744 +.. _Pull Request 12747: https://github.com/galaxyproject/galaxy/pull/12747 +.. _Pull Request 12752: https://github.com/galaxyproject/galaxy/pull/12752 +.. _Pull Request 12754: https://github.com/galaxyproject/galaxy/pull/12754 +.. _Pull Request 12755: https://github.com/galaxyproject/galaxy/pull/12755 +.. _Pull Request 12756: https://github.com/galaxyproject/galaxy/pull/12756 +.. _Pull Request 12759: https://github.com/galaxyproject/galaxy/pull/12759 +.. _Pull Request 12765: https://github.com/galaxyproject/galaxy/pull/12765 +.. _Pull Request 12773: https://github.com/galaxyproject/galaxy/pull/12773 +.. _Pull Request 12774: https://github.com/galaxyproject/galaxy/pull/12774 +.. _Pull Request 12778: https://github.com/galaxyproject/galaxy/pull/12778 +.. _Pull Request 12780: https://github.com/galaxyproject/galaxy/pull/12780 +.. _Pull Request 12781: https://github.com/galaxyproject/galaxy/pull/12781 +.. _Pull Request 12783: https://github.com/galaxyproject/galaxy/pull/12783 +.. _Pull Request 12787: https://github.com/galaxyproject/galaxy/pull/12787 +.. _Pull Request 12788: https://github.com/galaxyproject/galaxy/pull/12788 +.. _Pull Request 12791: https://github.com/galaxyproject/galaxy/pull/12791 +.. _Pull Request 12797: https://github.com/galaxyproject/galaxy/pull/12797 +.. _Pull Request 12801: https://github.com/galaxyproject/galaxy/pull/12801 +.. _Pull Request 12802: https://github.com/galaxyproject/galaxy/pull/12802 +.. _Pull Request 12810: https://github.com/galaxyproject/galaxy/pull/12810 +.. _Pull Request 12811: https://github.com/galaxyproject/galaxy/pull/12811 +.. _Pull Request 12812: https://github.com/galaxyproject/galaxy/pull/12812 +.. _Pull Request 12813: https://github.com/galaxyproject/galaxy/pull/12813 +.. _Pull Request 12819: https://github.com/galaxyproject/galaxy/pull/12819 +.. _Pull Request 12820: https://github.com/galaxyproject/galaxy/pull/12820 +.. _Pull Request 12826: https://github.com/galaxyproject/galaxy/pull/12826 +.. _Pull Request 12829: https://github.com/galaxyproject/galaxy/pull/12829 +.. _Pull Request 12831: https://github.com/galaxyproject/galaxy/pull/12831 +.. _Pull Request 12832: https://github.com/galaxyproject/galaxy/pull/12832 +.. _Pull Request 12833: https://github.com/galaxyproject/galaxy/pull/12833 +.. _Pull Request 12836: https://github.com/galaxyproject/galaxy/pull/12836 +.. _Pull Request 12838: https://github.com/galaxyproject/galaxy/pull/12838 +.. _Pull Request 12840: https://github.com/galaxyproject/galaxy/pull/12840 +.. _Pull Request 12841: https://github.com/galaxyproject/galaxy/pull/12841 +.. _Pull Request 12843: https://github.com/galaxyproject/galaxy/pull/12843 +.. _Pull Request 12844: https://github.com/galaxyproject/galaxy/pull/12844 +.. _Pull Request 12845: https://github.com/galaxyproject/galaxy/pull/12845 +.. _Pull Request 12847: https://github.com/galaxyproject/galaxy/pull/12847 +.. _Pull Request 12849: https://github.com/galaxyproject/galaxy/pull/12849 +.. _Pull Request 12851: https://github.com/galaxyproject/galaxy/pull/12851 +.. _Pull Request 12856: https://github.com/galaxyproject/galaxy/pull/12856 +.. _Pull Request 12858: https://github.com/galaxyproject/galaxy/pull/12858 +.. _Pull Request 12860: https://github.com/galaxyproject/galaxy/pull/12860 +.. _Pull Request 12861: https://github.com/galaxyproject/galaxy/pull/12861 +.. _Pull Request 12867: https://github.com/galaxyproject/galaxy/pull/12867 +.. _Pull Request 12868: https://github.com/galaxyproject/galaxy/pull/12868 +.. _Pull Request 12883: https://github.com/galaxyproject/galaxy/pull/12883 +.. _Pull Request 12885: https://github.com/galaxyproject/galaxy/pull/12885 +.. _Pull Request 12886: https://github.com/galaxyproject/galaxy/pull/12886 +.. _Pull Request 12888: https://github.com/galaxyproject/galaxy/pull/12888 +.. _Pull Request 12889: https://github.com/galaxyproject/galaxy/pull/12889 +.. _Pull Request 12890: https://github.com/galaxyproject/galaxy/pull/12890 +.. _Pull Request 12891: https://github.com/galaxyproject/galaxy/pull/12891 +.. _Pull Request 12892: https://github.com/galaxyproject/galaxy/pull/12892 +.. _Pull Request 12893: https://github.com/galaxyproject/galaxy/pull/12893 +.. _Pull Request 12894: https://github.com/galaxyproject/galaxy/pull/12894 +.. _Pull Request 12895: https://github.com/galaxyproject/galaxy/pull/12895 +.. _Pull Request 12896: https://github.com/galaxyproject/galaxy/pull/12896 +.. _Pull Request 12897: https://github.com/galaxyproject/galaxy/pull/12897 +.. _Pull Request 12898: https://github.com/galaxyproject/galaxy/pull/12898 +.. _Pull Request 12899: https://github.com/galaxyproject/galaxy/pull/12899 +.. _Pull Request 12900: https://github.com/galaxyproject/galaxy/pull/12900 +.. _Pull Request 12902: https://github.com/galaxyproject/galaxy/pull/12902 +.. _Pull Request 12904: https://github.com/galaxyproject/galaxy/pull/12904 +.. _Pull Request 12905: https://github.com/galaxyproject/galaxy/pull/12905 +.. _Pull Request 12906: https://github.com/galaxyproject/galaxy/pull/12906 +.. _Pull Request 12908: https://github.com/galaxyproject/galaxy/pull/12908 +.. _Pull Request 12910: https://github.com/galaxyproject/galaxy/pull/12910 +.. _Pull Request 12912: https://github.com/galaxyproject/galaxy/pull/12912 +.. _Pull Request 12913: https://github.com/galaxyproject/galaxy/pull/12913 +.. _Pull Request 12914: https://github.com/galaxyproject/galaxy/pull/12914 +.. _Pull Request 12915: https://github.com/galaxyproject/galaxy/pull/12915 +.. _Pull Request 12916: https://github.com/galaxyproject/galaxy/pull/12916 +.. _Pull Request 12919: https://github.com/galaxyproject/galaxy/pull/12919 +.. _Pull Request 12920: https://github.com/galaxyproject/galaxy/pull/12920 +.. _Pull Request 12921: https://github.com/galaxyproject/galaxy/pull/12921 +.. _Pull Request 12923: https://github.com/galaxyproject/galaxy/pull/12923 +.. _Pull Request 12924: https://github.com/galaxyproject/galaxy/pull/12924 +.. _Pull Request 12926: https://github.com/galaxyproject/galaxy/pull/12926 +.. _Pull Request 12929: https://github.com/galaxyproject/galaxy/pull/12929 +.. _Pull Request 12930: https://github.com/galaxyproject/galaxy/pull/12930 +.. _Pull Request 12931: https://github.com/galaxyproject/galaxy/pull/12931 +.. _Pull Request 12932: https://github.com/galaxyproject/galaxy/pull/12932 +.. _Pull Request 12935: https://github.com/galaxyproject/galaxy/pull/12935 +.. _Pull Request 12936: https://github.com/galaxyproject/galaxy/pull/12936 +.. _Pull Request 12937: https://github.com/galaxyproject/galaxy/pull/12937 +.. _Pull Request 12939: https://github.com/galaxyproject/galaxy/pull/12939 +.. _Pull Request 12940: https://github.com/galaxyproject/galaxy/pull/12940 +.. _Pull Request 12942: https://github.com/galaxyproject/galaxy/pull/12942 +.. _Pull Request 12944: https://github.com/galaxyproject/galaxy/pull/12944 +.. _Pull Request 12945: https://github.com/galaxyproject/galaxy/pull/12945 +.. _Pull Request 12946: https://github.com/galaxyproject/galaxy/pull/12946 +.. _Pull Request 12947: https://github.com/galaxyproject/galaxy/pull/12947 +.. _Pull Request 12948: https://github.com/galaxyproject/galaxy/pull/12948 +.. _Pull Request 12949: https://github.com/galaxyproject/galaxy/pull/12949 +.. _Pull Request 12950: https://github.com/galaxyproject/galaxy/pull/12950 +.. _Pull Request 12951: https://github.com/galaxyproject/galaxy/pull/12951 +.. _Pull Request 12954: https://github.com/galaxyproject/galaxy/pull/12954 +.. _Pull Request 12956: https://github.com/galaxyproject/galaxy/pull/12956 +.. _Pull Request 12960: https://github.com/galaxyproject/galaxy/pull/12960 +.. _Pull Request 12962: https://github.com/galaxyproject/galaxy/pull/12962 +.. _Pull Request 12963: https://github.com/galaxyproject/galaxy/pull/12963 +.. _Pull Request 12965: https://github.com/galaxyproject/galaxy/pull/12965 +.. _Pull Request 12966: https://github.com/galaxyproject/galaxy/pull/12966 +.. _Pull Request 12968: https://github.com/galaxyproject/galaxy/pull/12968 +.. _Pull Request 12969: https://github.com/galaxyproject/galaxy/pull/12969 +.. _Pull Request 12972: https://github.com/galaxyproject/galaxy/pull/12972 +.. _Pull Request 12973: https://github.com/galaxyproject/galaxy/pull/12973 +.. _Pull Request 12974: https://github.com/galaxyproject/galaxy/pull/12974 +.. _Pull Request 12976: https://github.com/galaxyproject/galaxy/pull/12976 +.. _Pull Request 12977: https://github.com/galaxyproject/galaxy/pull/12977 +.. _Pull Request 12978: https://github.com/galaxyproject/galaxy/pull/12978 +.. _Pull Request 12980: https://github.com/galaxyproject/galaxy/pull/12980 +.. _Pull Request 12982: https://github.com/galaxyproject/galaxy/pull/12982 +.. _Pull Request 12983: https://github.com/galaxyproject/galaxy/pull/12983 +.. _Pull Request 12984: https://github.com/galaxyproject/galaxy/pull/12984 +.. _Pull Request 12985: https://github.com/galaxyproject/galaxy/pull/12985 +.. _Pull Request 12989: https://github.com/galaxyproject/galaxy/pull/12989 +.. _Pull Request 12990: https://github.com/galaxyproject/galaxy/pull/12990 +.. _Pull Request 12991: https://github.com/galaxyproject/galaxy/pull/12991 +.. _Pull Request 12993: https://github.com/galaxyproject/galaxy/pull/12993 +.. _Pull Request 12994: https://github.com/galaxyproject/galaxy/pull/12994 +.. _Pull Request 12996: https://github.com/galaxyproject/galaxy/pull/12996 +.. _Pull Request 13000: https://github.com/galaxyproject/galaxy/pull/13000 +.. _Pull Request 13002: https://github.com/galaxyproject/galaxy/pull/13002 +.. _Pull Request 13003: https://github.com/galaxyproject/galaxy/pull/13003 +.. _Pull Request 13007: https://github.com/galaxyproject/galaxy/pull/13007 +.. _Pull Request 13009: https://github.com/galaxyproject/galaxy/pull/13009 +.. _Pull Request 13010: https://github.com/galaxyproject/galaxy/pull/13010 +.. _Pull Request 13011: https://github.com/galaxyproject/galaxy/pull/13011 +.. _Pull Request 13013: https://github.com/galaxyproject/galaxy/pull/13013 +.. _Pull Request 13017: https://github.com/galaxyproject/galaxy/pull/13017 +.. _Pull Request 13018: https://github.com/galaxyproject/galaxy/pull/13018 +.. _Pull Request 13021: https://github.com/galaxyproject/galaxy/pull/13021 +.. _Pull Request 13024: https://github.com/galaxyproject/galaxy/pull/13024 +.. _Pull Request 13025: https://github.com/galaxyproject/galaxy/pull/13025 +.. _Pull Request 13026: https://github.com/galaxyproject/galaxy/pull/13026 +.. _Pull Request 13029: https://github.com/galaxyproject/galaxy/pull/13029 +.. _Pull Request 13032: https://github.com/galaxyproject/galaxy/pull/13032 +.. _Pull Request 13033: https://github.com/galaxyproject/galaxy/pull/13033 +.. _Pull Request 13034: https://github.com/galaxyproject/galaxy/pull/13034 +.. _Pull Request 13035: https://github.com/galaxyproject/galaxy/pull/13035 +.. _Pull Request 13036: https://github.com/galaxyproject/galaxy/pull/13036 +.. _Pull Request 13038: https://github.com/galaxyproject/galaxy/pull/13038 +.. _Pull Request 13039: https://github.com/galaxyproject/galaxy/pull/13039 +.. _Pull Request 13041: https://github.com/galaxyproject/galaxy/pull/13041 +.. _Pull Request 13044: https://github.com/galaxyproject/galaxy/pull/13044 +.. _Pull Request 13047: https://github.com/galaxyproject/galaxy/pull/13047 +.. _Pull Request 13048: https://github.com/galaxyproject/galaxy/pull/13048 +.. _Pull Request 13049: https://github.com/galaxyproject/galaxy/pull/13049 +.. _Pull Request 13050: https://github.com/galaxyproject/galaxy/pull/13050 +.. _Pull Request 13051: https://github.com/galaxyproject/galaxy/pull/13051 +.. _Pull Request 13055: https://github.com/galaxyproject/galaxy/pull/13055 +.. _Pull Request 13056: https://github.com/galaxyproject/galaxy/pull/13056 +.. _Pull Request 13057: https://github.com/galaxyproject/galaxy/pull/13057 +.. _Pull Request 13058: https://github.com/galaxyproject/galaxy/pull/13058 +.. _Pull Request 13059: https://github.com/galaxyproject/galaxy/pull/13059 +.. _Pull Request 13060: https://github.com/galaxyproject/galaxy/pull/13060 +.. _Pull Request 13061: https://github.com/galaxyproject/galaxy/pull/13061 +.. _Pull Request 13062: https://github.com/galaxyproject/galaxy/pull/13062 +.. _Pull Request 13063: https://github.com/galaxyproject/galaxy/pull/13063 +.. _Pull Request 13065: https://github.com/galaxyproject/galaxy/pull/13065 +.. _Pull Request 13067: https://github.com/galaxyproject/galaxy/pull/13067 +.. _Pull Request 13068: https://github.com/galaxyproject/galaxy/pull/13068 +.. _Pull Request 13069: https://github.com/galaxyproject/galaxy/pull/13069 +.. _Pull Request 13072: https://github.com/galaxyproject/galaxy/pull/13072 +.. _Pull Request 13073: https://github.com/galaxyproject/galaxy/pull/13073 +.. _Pull Request 13074: https://github.com/galaxyproject/galaxy/pull/13074 +.. _Pull Request 13076: https://github.com/galaxyproject/galaxy/pull/13076 +.. _Pull Request 13078: https://github.com/galaxyproject/galaxy/pull/13078 +.. _Pull Request 13080: https://github.com/galaxyproject/galaxy/pull/13080 +.. _Pull Request 13081: https://github.com/galaxyproject/galaxy/pull/13081 +.. _Pull Request 13082: https://github.com/galaxyproject/galaxy/pull/13082 +.. _Pull Request 13083: https://github.com/galaxyproject/galaxy/pull/13083 +.. _Pull Request 13086: https://github.com/galaxyproject/galaxy/pull/13086 +.. _Pull Request 13087: https://github.com/galaxyproject/galaxy/pull/13087 +.. _Pull Request 13088: https://github.com/galaxyproject/galaxy/pull/13088 +.. _Pull Request 13092: https://github.com/galaxyproject/galaxy/pull/13092 +.. _Pull Request 13093: https://github.com/galaxyproject/galaxy/pull/13093 +.. _Pull Request 13094: https://github.com/galaxyproject/galaxy/pull/13094 +.. _Pull Request 13096: https://github.com/galaxyproject/galaxy/pull/13096 +.. _Pull Request 13097: https://github.com/galaxyproject/galaxy/pull/13097 +.. _Pull Request 13098: https://github.com/galaxyproject/galaxy/pull/13098 +.. _Pull Request 13099: https://github.com/galaxyproject/galaxy/pull/13099 +.. _Pull Request 13102: https://github.com/galaxyproject/galaxy/pull/13102 +.. _Pull Request 13103: https://github.com/galaxyproject/galaxy/pull/13103 +.. _Pull Request 13104: https://github.com/galaxyproject/galaxy/pull/13104 +.. _Pull Request 13107: https://github.com/galaxyproject/galaxy/pull/13107 +.. _Pull Request 13112: https://github.com/galaxyproject/galaxy/pull/13112 +.. _Pull Request 13114: https://github.com/galaxyproject/galaxy/pull/13114 +.. _Pull Request 13116: https://github.com/galaxyproject/galaxy/pull/13116 +.. _Pull Request 13118: https://github.com/galaxyproject/galaxy/pull/13118 +.. _Pull Request 13120: https://github.com/galaxyproject/galaxy/pull/13120 +.. _Pull Request 13121: https://github.com/galaxyproject/galaxy/pull/13121 +.. _Pull Request 13123: https://github.com/galaxyproject/galaxy/pull/13123 +.. _Pull Request 13124: https://github.com/galaxyproject/galaxy/pull/13124 +.. _Pull Request 13125: https://github.com/galaxyproject/galaxy/pull/13125 +.. _Pull Request 13126: https://github.com/galaxyproject/galaxy/pull/13126 +.. _Pull Request 13127: https://github.com/galaxyproject/galaxy/pull/13127 +.. _Pull Request 13128: https://github.com/galaxyproject/galaxy/pull/13128 +.. _Pull Request 13129: https://github.com/galaxyproject/galaxy/pull/13129 +.. _Pull Request 13130: https://github.com/galaxyproject/galaxy/pull/13130 +.. _Pull Request 13134: https://github.com/galaxyproject/galaxy/pull/13134 +.. _Pull Request 13135: https://github.com/galaxyproject/galaxy/pull/13135 +.. _Pull Request 13137: https://github.com/galaxyproject/galaxy/pull/13137 +.. _Pull Request 13139: https://github.com/galaxyproject/galaxy/pull/13139 +.. _Pull Request 13142: https://github.com/galaxyproject/galaxy/pull/13142 +.. _Pull Request 13143: https://github.com/galaxyproject/galaxy/pull/13143 +.. _Pull Request 13147: https://github.com/galaxyproject/galaxy/pull/13147 +.. _Pull Request 13148: https://github.com/galaxyproject/galaxy/pull/13148 +.. _Pull Request 13151: https://github.com/galaxyproject/galaxy/pull/13151 +.. _Pull Request 13152: https://github.com/galaxyproject/galaxy/pull/13152 +.. _Pull Request 13154: https://github.com/galaxyproject/galaxy/pull/13154 +.. _Pull Request 13155: https://github.com/galaxyproject/galaxy/pull/13155 +.. _Pull Request 13156: https://github.com/galaxyproject/galaxy/pull/13156 +.. _Pull Request 13157: https://github.com/galaxyproject/galaxy/pull/13157 +.. _Pull Request 13159: https://github.com/galaxyproject/galaxy/pull/13159 +.. _Pull Request 13162: https://github.com/galaxyproject/galaxy/pull/13162 +.. _Pull Request 13163: https://github.com/galaxyproject/galaxy/pull/13163 +.. _Pull Request 13165: https://github.com/galaxyproject/galaxy/pull/13165 +.. _Pull Request 13166: https://github.com/galaxyproject/galaxy/pull/13166 +.. _Pull Request 13167: https://github.com/galaxyproject/galaxy/pull/13167 +.. _Pull Request 13170: https://github.com/galaxyproject/galaxy/pull/13170 +.. _Pull Request 13171: https://github.com/galaxyproject/galaxy/pull/13171 +.. _Pull Request 13174: https://github.com/galaxyproject/galaxy/pull/13174 +.. _Pull Request 13175: https://github.com/galaxyproject/galaxy/pull/13175 +.. _Pull Request 13176: https://github.com/galaxyproject/galaxy/pull/13176 +.. _Pull Request 13178: https://github.com/galaxyproject/galaxy/pull/13178 +.. _Pull Request 13179: https://github.com/galaxyproject/galaxy/pull/13179 +.. _Pull Request 13182: https://github.com/galaxyproject/galaxy/pull/13182 +.. _Pull Request 13185: https://github.com/galaxyproject/galaxy/pull/13185 +.. _Pull Request 13186: https://github.com/galaxyproject/galaxy/pull/13186 +.. _Pull Request 13188: https://github.com/galaxyproject/galaxy/pull/13188 +.. _Pull Request 13189: https://github.com/galaxyproject/galaxy/pull/13189 +.. _Pull Request 13193: https://github.com/galaxyproject/galaxy/pull/13193 +.. _Pull Request 13196: https://github.com/galaxyproject/galaxy/pull/13196 +.. _Pull Request 13202: https://github.com/galaxyproject/galaxy/pull/13202 +.. _Pull Request 13203: https://github.com/galaxyproject/galaxy/pull/13203 +.. _Pull Request 13204: https://github.com/galaxyproject/galaxy/pull/13204 +.. _Pull Request 13205: https://github.com/galaxyproject/galaxy/pull/13205 +.. _Pull Request 13206: https://github.com/galaxyproject/galaxy/pull/13206 +.. _Pull Request 13207: https://github.com/galaxyproject/galaxy/pull/13207 +.. _Pull Request 13208: https://github.com/galaxyproject/galaxy/pull/13208 +.. _Pull Request 13209: https://github.com/galaxyproject/galaxy/pull/13209 +.. _Pull Request 13214: https://github.com/galaxyproject/galaxy/pull/13214 +.. _Pull Request 13215: https://github.com/galaxyproject/galaxy/pull/13215 +.. _Pull Request 13216: https://github.com/galaxyproject/galaxy/pull/13216 +.. _Pull Request 13217: https://github.com/galaxyproject/galaxy/pull/13217 +.. _Pull Request 13218: https://github.com/galaxyproject/galaxy/pull/13218 +.. _Pull Request 13219: https://github.com/galaxyproject/galaxy/pull/13219 +.. _Pull Request 13223: https://github.com/galaxyproject/galaxy/pull/13223 +.. _Pull Request 13224: https://github.com/galaxyproject/galaxy/pull/13224 +.. _Pull Request 13225: https://github.com/galaxyproject/galaxy/pull/13225 +.. _Pull Request 13226: https://github.com/galaxyproject/galaxy/pull/13226 +.. _Pull Request 13227: https://github.com/galaxyproject/galaxy/pull/13227 +.. _Pull Request 13228: https://github.com/galaxyproject/galaxy/pull/13228 +.. _Pull Request 13230: https://github.com/galaxyproject/galaxy/pull/13230 +.. _Pull Request 13231: https://github.com/galaxyproject/galaxy/pull/13231 +.. _Pull Request 13233: https://github.com/galaxyproject/galaxy/pull/13233 +.. _Pull Request 13236: https://github.com/galaxyproject/galaxy/pull/13236 +.. _Pull Request 13237: https://github.com/galaxyproject/galaxy/pull/13237 +.. _Pull Request 13238: https://github.com/galaxyproject/galaxy/pull/13238 +.. _Pull Request 13246: https://github.com/galaxyproject/galaxy/pull/13246 +.. _Pull Request 13249: https://github.com/galaxyproject/galaxy/pull/13249 +.. _Pull Request 13250: https://github.com/galaxyproject/galaxy/pull/13250 +.. _Pull Request 13251: https://github.com/galaxyproject/galaxy/pull/13251 +.. _Pull Request 13252: https://github.com/galaxyproject/galaxy/pull/13252 +.. _Pull Request 13254: https://github.com/galaxyproject/galaxy/pull/13254 +.. _Pull Request 13260: https://github.com/galaxyproject/galaxy/pull/13260 +.. _Pull Request 13261: https://github.com/galaxyproject/galaxy/pull/13261 +.. _Pull Request 13265: https://github.com/galaxyproject/galaxy/pull/13265 +.. _Pull Request 13267: https://github.com/galaxyproject/galaxy/pull/13267 +.. _Pull Request 13268: https://github.com/galaxyproject/galaxy/pull/13268 +.. _Pull Request 13271: https://github.com/galaxyproject/galaxy/pull/13271 +.. _Pull Request 13273: https://github.com/galaxyproject/galaxy/pull/13273 +.. _Pull Request 13275: https://github.com/galaxyproject/galaxy/pull/13275 +.. _Pull Request 13278: https://github.com/galaxyproject/galaxy/pull/13278 +.. _Pull Request 13281: https://github.com/galaxyproject/galaxy/pull/13281 +.. _Pull Request 13285: https://github.com/galaxyproject/galaxy/pull/13285 +.. _Pull Request 13290: https://github.com/galaxyproject/galaxy/pull/13290 +.. _Pull Request 13292: https://github.com/galaxyproject/galaxy/pull/13292 +.. _Pull Request 13293: https://github.com/galaxyproject/galaxy/pull/13293 +.. _Pull Request 13299: https://github.com/galaxyproject/galaxy/pull/13299 +.. _Pull Request 13301: https://github.com/galaxyproject/galaxy/pull/13301 +.. _Pull Request 13304: https://github.com/galaxyproject/galaxy/pull/13304 +.. _Pull Request 13309: https://github.com/galaxyproject/galaxy/pull/13309 +.. _Pull Request 13313: https://github.com/galaxyproject/galaxy/pull/13313 +.. _Pull Request 13315: https://github.com/galaxyproject/galaxy/pull/13315 +.. _Pull Request 13316: https://github.com/galaxyproject/galaxy/pull/13316 +.. _Pull Request 13317: https://github.com/galaxyproject/galaxy/pull/13317 +.. _Pull Request 13322: https://github.com/galaxyproject/galaxy/pull/13322 +.. _Pull Request 13324: https://github.com/galaxyproject/galaxy/pull/13324 +.. _Pull Request 13325: https://github.com/galaxyproject/galaxy/pull/13325 +.. _Pull Request 13330: https://github.com/galaxyproject/galaxy/pull/13330 +.. _Pull Request 13333: https://github.com/galaxyproject/galaxy/pull/13333 +.. _Pull Request 13343: https://github.com/galaxyproject/galaxy/pull/13343 +.. _Pull Request 13347: https://github.com/galaxyproject/galaxy/pull/13347 +.. _Pull Request 13363: https://github.com/galaxyproject/galaxy/pull/13363 +.. _Pull Request 13373: https://github.com/galaxyproject/galaxy/pull/13373 +.. _Pull Request 13381: https://github.com/galaxyproject/galaxy/pull/13381 +.. _Pull Request 13385: https://github.com/galaxyproject/galaxy/pull/13385 +.. _Pull Request 13389: https://github.com/galaxyproject/galaxy/pull/13389 +.. _Pull Request 13390: https://github.com/galaxyproject/galaxy/pull/13390 +.. _Pull Request 13392: https://github.com/galaxyproject/galaxy/pull/13392 +.. _Pull Request 13396: https://github.com/galaxyproject/galaxy/pull/13396 +.. _Pull Request 13403: https://github.com/galaxyproject/galaxy/pull/13403 +.. _Pull Request 13407: https://github.com/galaxyproject/galaxy/pull/13407 +.. _Pull Request 13417: https://github.com/galaxyproject/galaxy/pull/13417 +.. _Pull Request 13420: https://github.com/galaxyproject/galaxy/pull/13420 +.. _Pull Request 13425: https://github.com/galaxyproject/galaxy/pull/13425 +.. _Pull Request 13429: https://github.com/galaxyproject/galaxy/pull/13429 +.. _Pull Request 13430: https://github.com/galaxyproject/galaxy/pull/13430 +.. _Pull Request 13432: https://github.com/galaxyproject/galaxy/pull/13432 +.. _Pull Request 13433: https://github.com/galaxyproject/galaxy/pull/13433 +.. _Pull Request 13436: https://github.com/galaxyproject/galaxy/pull/13436 +.. _Pull Request 13439: https://github.com/galaxyproject/galaxy/pull/13439 +.. _Pull Request 13440: https://github.com/galaxyproject/galaxy/pull/13440 +.. _Pull Request 13441: https://github.com/galaxyproject/galaxy/pull/13441 +.. _Pull Request 13442: https://github.com/galaxyproject/galaxy/pull/13442 +.. _Pull Request 13454: https://github.com/galaxyproject/galaxy/pull/13454 +.. _Pull Request 13461: https://github.com/galaxyproject/galaxy/pull/13461 +.. _Pull Request 13462: https://github.com/galaxyproject/galaxy/pull/13462 +.. _Pull Request 13468: https://github.com/galaxyproject/galaxy/pull/13468 +.. _Pull Request 13469: https://github.com/galaxyproject/galaxy/pull/13469 +.. _Pull Request 13470: https://github.com/galaxyproject/galaxy/pull/13470 +.. _Pull Request 13474: https://github.com/galaxyproject/galaxy/pull/13474 +.. _Pull Request 13478: https://github.com/galaxyproject/galaxy/pull/13478 +.. _Pull Request 13480: https://github.com/galaxyproject/galaxy/pull/13480 +.. _Pull Request 13483: https://github.com/galaxyproject/galaxy/pull/13483 +.. _Pull Request 13484: https://github.com/galaxyproject/galaxy/pull/13484 +.. _Pull Request 13485: https://github.com/galaxyproject/galaxy/pull/13485 diff --git a/doc/source/releases/22.05_announce.rst b/doc/source/releases/22.05_announce.rst new file mode 100644 index 00000000000..1997de3e400 --- /dev/null +++ b/doc/source/releases/22.05_announce.rst @@ -0,0 +1,12 @@ + +:orphan: + +=========================================================== +May 2022 Galaxy Release (v 22.05) +=========================================================== + + +Schedule +=========================================================== + * Planned Freeze Date: 2022-05-02 + * Planned Release Date: 2022-05-23 diff --git a/doc/source/releases/images/22.01-colour.mp4 b/doc/source/releases/images/22.01-colour.mp4 new file mode 100644 index 00000000000..b8ca5236894 Binary files /dev/null and b/doc/source/releases/images/22.01-colour.mp4 differ diff --git a/doc/source/releases/images/22.01-export.mp4 b/doc/source/releases/images/22.01-export.mp4 new file mode 100644 index 00000000000..e832b0a2948 Binary files /dev/null and b/doc/source/releases/images/22.01-export.mp4 differ diff --git a/doc/source/releases/index.rst b/doc/source/releases/index.rst index ce5f17b1faf..da1e2921287 100644 --- a/doc/source/releases/index.rst +++ b/doc/source/releases/index.rst @@ -4,6 +4,7 @@ Releases .. toctree:: :maxdepth: 1 + 22.01_announce_user 21.09_announce_user 21.05_announce_user 21.01_announce_user diff --git a/lib/galaxy/model/orm/engine_factory.py b/lib/galaxy/model/orm/engine_factory.py index 0c67546ce35..933f52c607c 100644 --- a/lib/galaxy/model/orm/engine_factory.py +++ b/lib/galaxy/model/orm/engine_factory.py @@ -8,6 +8,7 @@ from multiprocessing.util import register_after_fork from sqlalchemy import ( create_engine, event, + exc, ) from sqlalchemy.engine import Engine @@ -106,5 +107,22 @@ def build_engine( # Create the database engine engine_options = engine_options or {} engine = create_engine(url, connect_args=connect_args, **engine_options) + + # Prevent sharing connection across fork: https://docs.sqlalchemy.org/en/14/core/pooling.html#using-connection-pools-with-multiprocessing-or-os-fork register_after_fork(engine, lambda e: e.dispose()) + + @event.listens_for(engine, "connect") + def connect(dbapi_connection, connection_record): + connection_record.info["pid"] = os.getpid() + + @event.listens_for(engine, "checkout") + def checkout(dbapi_connection, connection_record, connection_proxy): + pid = os.getpid() + if connection_record.info["pid"] != pid: + connection_record.dbapi_connection = connection_proxy.dbapi_connection = None + raise exc.DisconnectionError( + "Connection record belongs to pid %s, " + "attempting to check out in pid %s" % (connection_record.info["pid"], pid) + ) + return engine diff --git a/lib/galaxy/model/unittest_utils/data_app.py b/lib/galaxy/model/unittest_utils/data_app.py index 5ff5dc3de92..e081cf0eabe 100644 --- a/lib/galaxy/model/unittest_utils/data_app.py +++ b/lib/galaxy/model/unittest_utils/data_app.py @@ -61,6 +61,7 @@ class GalaxyDataTestConfig(Bunch): self.jobs_directory = os.path.join(self.data_dir, "jobs_directory") self.new_file_path = os.path.join(self.data_dir, "tmp") self.file_path = os.path.join(self.data_dir, "files") + self.server_name = "main" def __del__(self): if self._remove_root: diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py index 58d29565c4b..e47b3af9af6 100644 --- a/lib/galaxy/selenium/navigates_galaxy.py +++ b/lib/galaxy/selenium/navigates_galaxy.py @@ -1243,7 +1243,7 @@ class NavigatesGalaxy(HasDriver): self.send_enter(tag_area) def workflow_run_submit(self): - self.wait_for_and_click_selector("#run-workflow") + self.wait_for_and_click_selector("button.btn-primary") def workflow_create_new(self, annotation=None, clear_placeholder=False): self.workflow_index_open() diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 39feb8ba5bf..0fe105665bd 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -333,7 +333,11 @@ class SimpleTextToolParameter(ToolParameter): def to_json(self, value, app, use_security): """Convert a value to a string representation suitable for persisting""" - return unicodify(value) + if value is None: + rval = "" if not self.optional else None + else: + rval = unicodify(value) + return rval def get_initial_value(self, trans, other_values): return self.value diff --git a/lib/galaxy/tools/parameters/validation.py b/lib/galaxy/tools/parameters/validation.py index ae38ca7db93..f907bde7857 100644 --- a/lib/galaxy/tools/parameters/validation.py +++ b/lib/galaxy/tools/parameters/validation.py @@ -742,9 +742,7 @@ class EmptyTextfieldValidator(Validator): return cls(message, negate) def validate(self, value, trans=None): - # Ideally the empty string validator should only check against an actual empty string, - # but the fill_defaults logic will store the result of `get_initial_value``, which is None. - super().validate(value not in ("", None)) + super().validate(value != "") class MetadataInFileColumnValidator(Validator): diff --git a/lib/galaxy/web_stack/__init__.py b/lib/galaxy/web_stack/__init__.py index dca8c2af3d4..fc515c28647 100644 --- a/lib/galaxy/web_stack/__init__.py +++ b/lib/galaxy/web_stack/__init__.py @@ -5,6 +5,8 @@ import json import logging import multiprocessing import os +import sys +import threading from typing import ( Callable, Dict, @@ -213,6 +215,8 @@ class ApplicationStack: def set_postfork_server_name(self, app): new_server_name = self.server_name_template.format(**self.facts) + if "GUNICORN_WORKER_ID" in os.environ: + new_server_name = f"{new_server_name}.{os.environ['GUNICORN_WORKER_ID']}" multiprocessing.current_process().name = app.config.server_name = new_server_name log.debug("server_name set to: %s", new_server_name) @@ -584,6 +588,45 @@ class PasteApplicationStack(ApplicationStack): name = "Python Paste" +class GunicornApplicationStack(ApplicationStack): + name = "Gunicorn" + do_post_fork = "--preload" in os.environ.get("GUNICORN_CMD_ARGS", "") or "--preload" in sys.argv + postfork_functions: List[Callable] = [] + # Will be set to True by external hook + late_postfork_event = threading.Event() + + @classmethod + def register_postfork_function(cls, f, *args, **kwargs): + # do_post_fork determines if we need to run postfork functions + if cls.do_post_fork: + # if so, we call ApplicationStack.late_postfork once after forking ... + if not cls.postfork_functions: + os.register_at_fork(after_in_child=cls.late_postfork) + # ... and store everything we need to run in ApplicationStack.postfork_functions + cls.postfork_functions.append(lambda: f(*args, **kwargs)) + else: + f(*args, **kwargs) + + @classmethod + def run_postfork(cls): + cls.late_postfork_event.wait(1) + for f in cls.postfork_functions: + f() + + @classmethod + def late_postfork(cls): + # We can't run postfork functions immediately, because this is before the gunicorn `post_fork` hook runs, + # and we depend on the `post_fork` hook to set a worker id. + t = threading.Thread(target=cls.run_postfork) + t.start() + + def log_startup(self): + msg = [f"Galaxy server instance '{self.config.server_name}' is running"] + if "GUNICORN_LISTENERS" in os.environ: + msg.append(f'serving on {os.environ["GUNICORN_LISTENERS"]}') + log.info("\n".join(msg)) + + class WeblessApplicationStack(ApplicationStack): name = "Webless" @@ -641,7 +684,9 @@ def application_stack_class() -> Type[ApplicationStack]: """Returns the correct ApplicationStack class for the stack under which this Galaxy process is running. """ - if uwsgi is not None and hasattr(uwsgi, "numproc"): + if "gunicorn" in os.environ.get("SERVER_SOFTWARE", ""): + return GunicornApplicationStack + elif uwsgi is not None and hasattr(uwsgi, "numproc"): return UWSGIApplicationStack else: # cleverer ideas welcome diff --git a/lib/galaxy/web_stack/gunicorn_config.py b/lib/galaxy/web_stack/gunicorn_config.py new file mode 100644 index 00000000000..6f4249a8acd --- /dev/null +++ b/lib/galaxy/web_stack/gunicorn_config.py @@ -0,0 +1,67 @@ +""" +Gunicorn config file based on https://gist.github.com/hynek/ba655c8756924a5febc5285c712a7946 +""" +import logging +import os +import sys + +log = logging.getLogger(__name__) + + +def on_starting(server): + """ + Attach a set of IDs that can be temporarily re-used. + + Used on reloads when each worker exists twice. + """ + server._worker_id_overload = set() + + +def nworkers_changed(server, new_value, old_value): + """ + Gets called on startup too. + + Set the current number of workers. Required if we raise the worker count + temporarily using TTIN because server.cfg.workers won't be updated and if + one of those workers dies, we wouldn't know the ids go that far. + """ + server._worker_id_current_workers = new_value + + +def _next_worker_id(server): + """ + If there are IDs open for re-use, take one. Else look for a free one. + """ + if server._worker_id_overload: + return server._worker_id_overload.pop() + + in_use = set(w._worker_id for w in tuple(server.WORKERS.values()) if w.alive) + free = set(range(1, server._worker_id_current_workers + 1)) - in_use + + return free.pop() + + +def on_reload(server): + """ + Add a full set of ids into overload so it can be re-used once. + """ + server._worker_id_overload = set(range(1, server.cfg.workers + 1)) + + +def pre_fork(server, worker): + """ + Attach the next free worker_id before forking off. + """ + worker._worker_id = _next_worker_id(server) + + +def post_fork(server, worker): + """ + Put worker_id and listeners into an env variable for further use within the app. + """ + os.environ["GUNICORN_WORKER_ID"] = str(worker._worker_id) + os.environ["GUNICORN_LISTENERS"] = ",".join(str(bind) for bind in server.LISTENERS) + if "--preload" in os.environ.get("GUNICORN_CMD_ARGS", "") or "--preload" in sys.argv: + from galaxy.web_stack import GunicornApplicationStack + + GunicornApplicationStack.late_postfork_event.set() diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py index 95daff9565a..43160e314fe 100644 --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -484,7 +484,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi stored = self.get_stored_workflow(trans, id) if new_annotation: # Sanitize annotation before adding it. - new_annotation = sanitize_html(new_annotation or "") + new_annotation = sanitize_html(new_annotation) self.add_item_annotation(trans.sa_session, trans.get_user(), stored, new_annotation) trans.sa_session.flush() return new_annotation @@ -639,7 +639,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi workflow.stored_workflow = stored_workflow stored_workflow.latest_workflow = workflow # Add annotation. - workflow_annotation = sanitize_html(workflow_annotation or "") + workflow_annotation = sanitize_html(workflow_annotation) self.add_item_annotation(trans.sa_session, trans.get_user(), stored_workflow, workflow_annotation) # Persist session = trans.sa_session @@ -668,7 +668,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi workflow.stored_workflow = stored_workflow stored_workflow.latest_workflow = workflow # Add annotation. - workflow_annotation = sanitize_html(workflow_annotation or "") + workflow_annotation = sanitize_html(workflow_annotation) self.add_item_annotation(trans.sa_session, trans.get_user(), stored_workflow, workflow_annotation) # Persist diff --git a/lib/galaxy/webapps/galaxy/fast_factory.py b/lib/galaxy/webapps/galaxy/fast_factory.py index d2210e6f3fd..fa386c0380a 100644 --- a/lib/galaxy/webapps/galaxy/fast_factory.py +++ b/lib/galaxy/webapps/galaxy/fast_factory.py @@ -37,7 +37,7 @@ with the following command-line. :: - gunicorn 'galaxy.webapps.galaxy.fast_factory:factory()' --env GALAXY_CONFIG_FILE=config/galaxy.ini --pythonpath lib -w 4 -k uvicorn.workers.UvicornWorker + gunicorn 'galaxy.webapps.galaxy.fast_factory:factory()' --env GALAXY_CONFIG_FILE=config/galaxy.ini --pythonpath lib -w 4 -k uvicorn.workers.UvicornWorker --config lib/galaxy/web_stack/gunicorn_config.py """ diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 89e170899ab..ddb2f73091a 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -925,9 +925,9 @@ class InputParameterModule(WorkflowModule): ) if param_type == "text": if parameter_type == "text": - text_default = parameter_def.get("default") + text_default = parameter_def.get("default") or "" else: - text_default = None + text_default = "" default_source["value"] = text_default input_default_value: Union[ TextToolParameter, diff --git a/lib/galaxy/workflow/run_request.py b/lib/galaxy/workflow/run_request.py index 1c9e2fa97d5..6cb858ac948 100644 --- a/lib/galaxy/workflow/run_request.py +++ b/lib/galaxy/workflow/run_request.py @@ -94,17 +94,10 @@ def _normalize_inputs(steps, inputs, inputs_by): # start to ensure tool state is being preserved and loaded in a type safe way. assert isinstance(optional, bool) if not inputs_key and default_value is None and not optional: - message = f"Workflow cannot be run because an expected input step '{step.order_index}' ({step.label}) is not optional and no input." + message = f"Workflow cannot be run because an expected input step '{step.id}' ({step.label}) is not optional and no input." raise exceptions.MessageException(message) if inputs_key: - input_value = inputs[inputs_key] - if input_value is None and default_value is None and not optional: - # We've got a null value in a required parameter. - # I can't think of a situation where this should be valid, - # so let's fail early here - message = f"Workflow cannot be run because an expected input step '{step.order_index}' ({step.label}) is not optional and input is null." - raise exceptions.MessageException(message) - normalized_inputs[step.id] = input_value + normalized_inputs[step.id] = inputs[inputs_key] return normalized_inputs diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index 8ba9298a2f3..6795da10c5d 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -2917,43 +2917,6 @@ text_input: jobs = self._history_jobs(history_id) assert len(jobs) == 1 - def test_run_with_required_text_parameter_not_provided_fails(self): - with self.dataset_populator.test_history() as history_id: - try: - self._run_workflow( - """ -class: GalaxyWorkflow -inputs: - text_input: text -steps: [] -""", - assert_ok=True, - history_id=history_id, - ) - except AssertionError as e: - assert "(text_input) is not optional" in str(e) - - def test_run_with_required_text_parameter_null_fails(self): - with self.dataset_populator.test_history() as history_id: - try: - self._run_workflow( - """ -class: GalaxyWorkflow -inputs: - text_input: text -steps: [] -""", - test_data=""" -text_input: - value: null - type: raw -""", - assert_ok=True, - history_id=history_id, - ) - except AssertionError as e: - assert "(text_input) is not optional" in str(e) - def test_run_with_int_parameter(self): with self.dataset_populator.test_history() as history_id: failed = False diff --git a/lib/galaxy_test/selenium/test_workflow_run.py b/lib/galaxy_test/selenium/test_workflow_run.py index 208df75e14a..28cb9aa82af 100644 --- a/lib/galaxy_test/selenium/test_workflow_run.py +++ b/lib/galaxy_test/selenium/test_workflow_run.py @@ -258,21 +258,6 @@ steps: history_id = self.current_history_id() self.workflow_populator.wait_for_history_workflows(history_id, expected_invocation_count=1) - @selenium_test - def test_simple_workflow_run_form_validation_required_text_param(self): - workflow_id = self.workflow_populator.upload_yaml_workflow( - """ -class: GalaxyWorkflow -inputs: - text_input: text -steps: [] -""" - ) - self.simplified_workflow_run_by_id(workflow_id) - run_button = self.components.workflow_run.run_workflow.wait_for_visible() - assert run_button.get_attribute("disabled") == "true" - self.components.workflow_run.validation_error.wait_for_visible() - def open_in_workflow_run(self, yaml_content): name = self.workflow_upload_yaml_with_random_name(yaml_content) self.workflow_run_with_name(name) @@ -299,9 +284,6 @@ steps: [] self.workflow_index_search_for(name) self.workflow_click_option(".workflow-run") - def simplified_workflow_run_by_id(self, workflow_id): - self.get(f"workflows/run?id={workflow_id}&simplified_workflow_run_ui=prefer") - def _assert_has_3_lines_after_run(self, hid): if self.is_beta_history(): self.content_item_by_attributes(hid=hid, state="ok").wait_for_present() diff --git a/lib/tool_shed/webapp/fast_factory.py b/lib/tool_shed/webapp/fast_factory.py index 6b7285dc0e1..73c9c40d625 100644 --- a/lib/tool_shed/webapp/fast_factory.py +++ b/lib/tool_shed/webapp/fast_factory.py @@ -30,7 +30,7 @@ with the following command-line. :: - gunicorn 'tool_shed.webapp.fast_factory:factory()' --env TOOL_SHED_CONFIG_FILE=config/tool_shed.yml --pythonpath lib -w 4 -k uvicorn.workers.UvicornWorker + gunicorn 'tool_shed.webapp.fast_factory:factory()' --env TOOL_SHED_CONFIG_FILE=config/tool_shed.yml --pythonpath lib -w 4 -k uvicorn.workers.UvicornWorker --config lib/galaxy/web_stack/gunicorn_config.py """ diff --git a/packages/web_stack/test-requirements.txt b/packages/web_stack/test-requirements.txt index e079f8a6038..527b6063f0f 100644 --- a/packages/web_stack/test-requirements.txt +++ b/packages/web_stack/test-requirements.txt @@ -1 +1,2 @@ +gunicorn pytest diff --git a/scripts/common_startup_functions.sh b/scripts/common_startup_functions.sh index 89cfef3c8c1..118d4a5f875 100644 --- a/scripts/common_startup_functions.sh +++ b/scripts/common_startup_functions.sh @@ -163,10 +163,10 @@ find_server() { ;; reports) # TODO: is this really the only way to configure the port? - GUNICORN_CMD_ARGS=${GUNICORN_CMD_ARGS:-"--bind=localhost:9001"} + GUNICORN_CMD_ARGS=${GUNICORN_CMD_ARGS:-"--bind=localhost:9001 --config lib/galaxy/web_stack/gunicorn_config.py"} ;; tool_shed) - GUNICORN_CMD_ARGS=${GUNICORN_CMD_ARGS:-"--bind=localhost:9009"} + GUNICORN_CMD_ARGS=${GUNICORN_CMD_ARGS:-"--bind=localhost:9009 --config lib/galaxy/web_stack/gunicorn_config.py"} esac APP_WEBSERVER=${APP_WEBSERVER:-$default_webserver} diff --git a/test/unit/web_stack/test_application_stack.py b/test/unit/web_stack/test_application_stack.py new file mode 100644 index 00000000000..2e950dc8a84 --- /dev/null +++ b/test/unit/web_stack/test_application_stack.py @@ -0,0 +1,38 @@ +from gunicorn import SERVER_SOFTWARE + +from galaxy.model.unittest_utils import GalaxyDataTestApp +from galaxy.tool_util.verify.wait import wait_on +from galaxy.web_stack import ( + application_stack_class, + application_stack_instance, + GunicornApplicationStack, + WeblessApplicationStack, +) + + +def test_application_stack_class_factory(monkeypatch): + assert application_stack_class() == WeblessApplicationStack + monkeypatch.setenv("SERVER_SOFTWARE", SERVER_SOFTWARE) + assert application_stack_class() == GunicornApplicationStack + + +def test_application_stack_instance_factory(monkeypatch): + app = GalaxyDataTestApp() + monkeypatch.setenv("SERVER_SOFTWARE", SERVER_SOFTWARE) + stack_instance = application_stack_instance(app=app, config=app.config) + assert isinstance(stack_instance, GunicornApplicationStack) + assert stack_instance.name == "Gunicorn" + + +def test_application_stack_post_fork(monkeypatch): + app = GalaxyDataTestApp() + monkeypatch.setenv("SERVER_SOFTWARE", SERVER_SOFTWARE) + monkeypatch.setenv("GUNICORN_WORKER_ID", "100") + monkeypatch.setattr(GunicornApplicationStack, "do_post_fork", True) + stack_instance = application_stack_instance(app=app, config=app.config) + assert isinstance(stack_instance, GunicornApplicationStack) + stack_instance.register_postfork_function(lambda: stack_instance.set_postfork_server_name(app)) + assert not app.config.server_name.endswith(".100") + stack_instance.late_postfork() + stack_instance.late_postfork_event.set() + wait_on(lambda: app.config.server_name.endswith(".100") or None, desc="server name to change", timeout=1, delta=0.1)