Merge branch 'dev' of https://github.com/galaxyproject/galaxy into bugfix/8563/invert-tooltip-reversed

.dockerignore

@@ -1,2 +1,3 @@
+.k8s_ci.Dockerfile
 .venv
 database

.gitignore

@@ -162,10 +162,7 @@ client/galaxy
 doc/build
 doc/schema.md
 doc/source/admin/config_logging_default_yaml.rst
-doc/source/api/api.rst
-doc/source/api/ts_api.rst
 doc/source/dev/schema.md
-doc/source/dev/schema.rst
 client/styleguide
 
 # webpack stats

.k8s_ci.Dockerfile

@@ -1,12 +1,18 @@
 # Stage 1:
 # - base: ubuntu (default) OR prebuilt image0
 # - install build tools
-# - run playbook (image0 avoids rerunning lengthy tasks)
+# - clone playbook
+# Stage 2: build galaxy client and server in parallel
+# Stage 2.1:
+# - run playbook with server build steps only
 # - remove build artifacts + files not needed in container
-# Stage 2:
+# Stage 2.2:
+# - run playbook with client build steps only
+# - remove build artifacts + files not needed in container
+# Stage 3:
 # - install python-virtualenv
 # - create galaxy user + group + directory
-# - copy galaxy files from stage 1
+# - copy galaxy files from stage 2.1 and 2.2
 # - finalize container (set path, user...)
 
 # Init ARGs
@@ -16,12 +22,14 @@ ARG SERVER_DIR=$ROOT_DIR/server
 # For much faster build time override this with image0 (Dockerfile.0 build):
 #   docker build --build-arg BASE=<image0 name>...
 ARG STAGE1_BASE=ubuntu:20.04
-ARG STAGE2_BASE=$STAGE1_BASE
+ARG FINAL_STAGE_BASE=$STAGE1_BASE
 # NOTE: the value of GALAXY_USER must be also hardcoded in COPY in final stage
 ARG GALAXY_USER=galaxy
 ARG GALAXY_PLAYBOOK_REPO=https://github.com/galaxyproject/galaxy-docker-k8s
 
-# Stage-1
+#======================================================
+# Stage 1 - Setup common requirements for build
+#======================================================
 FROM $STAGE1_BASE AS stage1
 ARG DEBIAN_FRONTEND=noninteractive
 ARG SERVER_DIR
@@ -61,7 +69,14 @@ RUN ansible-galaxy install -r requirements.yml -p roles --force-with-deps
 
 # Add Galaxy source code
 COPY . $SERVER_DIR/
-RUN ansible-playbook -i localhost, playbook.yml -v
+
+#======================================================
+# Stage 2.1 - Build galaxy server
+#======================================================
+FROM stage1 AS server_build
+ARG SERVER_DIR
+
+RUN ansible-playbook -i localhost, playbook.yml -v -e galaxy_build_client=False
 
 RUN cat /galaxy/server/lib/galaxy/dependencies/conditional-requirements.txt | grep psycopg2-binary | xargs /galaxy/server/.venv/bin/pip install
 
@@ -72,7 +87,6 @@ RUN git rev-parse HEAD > GITREVISION
 RUN rm -rf \
         .ci \
         .git \
-        .venv/bin/node \
         .venv/include/node \
         .venv/src/node* \
         doc \
@@ -81,8 +95,30 @@ RUN rm -rf \
 # Clean up *all* node_modules, including plugins.  Everything is already built+staged.
 RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
 
-# Stage-2
-FROM $STAGE2_BASE
+#======================================================
+# Stage 2.2 - Build galaxy client
+#======================================================
+FROM stage1 AS client_build
+ARG SERVER_DIR
+
+RUN ansible-playbook -i localhost, playbook.yml -v --tags "galaxy_build_client"
+
+WORKDIR $SERVER_DIR
+RUN rm -rf \
+        .ci \
+        .git \
+        .venv/include/node \
+        .venv/src/node* \
+        doc \
+        test \
+        test-data
+# Clean up *all* node_modules, including plugins.  Everything is already built+staged.
+RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
+
+#======================================================
+# Stage 3 - Build final image based on previous stages
+#======================================================
+FROM $FINAL_STAGE_BASE
 ARG DEBIAN_FRONTEND=noninteractive
 ARG ROOT_DIR
 ARG SERVER_DIR
@@ -115,7 +151,8 @@ RUN set -xe; \
 WORKDIR $ROOT_DIR
 # Copy galaxy files to final image
 # The chown value MUST be hardcoded (see #35018 at github.com/moby/moby)
-COPY --chown=galaxy:galaxy --from=stage1 $ROOT_DIR .
+COPY --chown=galaxy:galaxy --from=server_build $ROOT_DIR .
+COPY --chown=galaxy:galaxy --from=client_build $SERVER_DIR/static ./server/static
 
 WORKDIR $SERVER_DIR
 EXPOSE 8080

CONTRIBUTING.md

@@ -41,7 +41,10 @@ Git](https://swcarpentry.github.io/git-novice/) tutorial is a good place to
 start.  More learning resources are listed at
 https://help.github.com/en/github/getting-started-with-github/git-and-github-learning-resources
 
-1. Make sure you have a free [GitHub](https://github.com/) account.
+1. Make sure you have a free [GitHub](https://github.com/) account. To increase
+   the security of your account, we strongly recommend that you configure
+   [two-factor authentication](https://docs.github.com/en/github/authenticating-to-github/securing-your-account-with-two-factor-authentication-2fa).
+   Additionally, you may want to [sign your commits](https://docs.github.com/en/github/authenticating-to-github/managing-commit-signature-verification).
 
 2. Fork the [galaxy repository](https://github.com/galaxyproject/galaxy) on
    GitHub to make your changes.  To keep your copy up to date with respect to

Makefile

@@ -30,6 +30,9 @@ docs: ## Generate HTML documentation.
 	$(IN_VENV) $(MAKE) -C doc clean
 	$(IN_VENV) $(MAKE) -C doc html
 
+docs-develop: ## Fast doc generation and more warnings (for development)
+	$(IN_VENV) GALAXY_DOCS_SKIP_VIEW_CODE=1 SPHINXOPTS='-j 4' $(MAKE) -C doc html
+
 setup-venv:
 	if [ ! -f $(VENV)/bin/activate ]; then bash scripts/common_startup.sh --dev-wheels; fi
 

client/gulpfile.js

@@ -22,6 +22,7 @@ const PLUGIN_BUILD_IDS = [
     "openlayers",
     "openseadragon",
     "pv",
+    "nora",
 ];
 
 const paths = {

client/src/components/DatasetInformation/DatasetInformation.test.js

@@ -1,6 +1,7 @@
 import axios from "axios";
 import MockAdapter from "axios-mock-adapter";
 import { mount } from "@vue/test-utils";
+import { getLocalVue } from "jest/helpers";
 import DatasetInformation from "./DatasetInformation";
 import datasetResponse from "./testData/datasetResponse";
 import flushPromises from "flush-promises";
@@ -18,6 +19,8 @@ const mockDatasetProvider = {
     },
 };
 
+const localVue = getLocalVue();
+
 describe("DatasetInformation/DatasetInformation.vue", () => {
     let wrapper;
     let datasetInfoTable;
@@ -42,6 +45,7 @@ describe("DatasetInformation/DatasetInformation.vue", () => {
             stubs: {
                 DatasetProvider: mockDatasetProvider,
             },
+            localVue,
         });
         datasetInfoTable = wrapper.find("#dataset-details");
         await flushPromises();

client/src/components/FilesDialog/FilesInput.vue

@@ -1,5 +1,6 @@
 <template>
-    <b-form-input v-model="localValue" :placeholder="placeholder" @click="selectFile"> </b-form-input>
+    <b-form-input class="directory-form-input" v-model="localValue" :placeholder="placeholder" @click="selectFile">
+    </b-form-input>
 </template>
 
 <script>

client/src/components/History/HistoryTopNav.vue

@@ -35,7 +35,7 @@
                 key="use-legacy-history"
                 title="Return to legacy history panel"
                 icon="fas fa-exchange-alt"
-                @click="useLegacyHistoryPanel"
+                @click="switchToLegacyHistoryPanel"
             />
         </PriorityMenu>
     </div>
@@ -48,6 +48,7 @@ import HistorySelector from "./HistorySelector";
 import { createNewHistory } from "./model/queries";
 import { wipeDatabase, clearHistoryDateStore } from "./caching";
 import { legacyNavigationMixin } from "components/plugins";
+import { switchToLegacyHistoryPanel } from "./adapters/betaToggle";
 
 export default {
     mixins: [legacyNavigationMixin],
@@ -72,10 +73,7 @@ export default {
     methods: {
         ...mapActions("betaHistory", ["storeHistory"]),
 
-        useLegacyHistoryPanel() {
-            sessionStorage.removeItem("useBetaHistory");
-            location.reload();
-        },
+        switchToLegacyHistoryPanel,
 
         // create new server, store in vuex, switch to it
         async createHistory() {

client/src/components/History/adapters/HistoryPanelProxy.js

@@ -15,6 +15,7 @@ import Backbone from "backbone";
 import CurrentHistoryView from "mvc/history/history-view-edit-current";
 import { History } from "mvc/history/history-model";
 import "./HistoryPanelProxy.scss";
+import store from "store";
 
 // bypass polling while using the beta panel, skips contents loading
 const FakeHistoryViewModel = CurrentHistoryView.CurrentHistoryView.extend({
@@ -37,6 +38,7 @@ export const HistoryPanelProxy = Backbone.View.extend({
 
         // fake view of the current history
         this.historyView = new FakeHistoryViewModel({
+            fakeHistoryViewModel: true,
             className: `fake ${CurrentHistoryView.CurrentHistoryView.prototype.className} middle`,
             purgeAllowed: this.allow_user_dataset_purge,
             linkTarget: "galaxy_main",
@@ -59,6 +61,14 @@ export const HistoryPanelProxy = Backbone.View.extend({
                 url: `${Galaxy.user.urlRoot()}/${Galaxy.user.id || "current"}`,
             });
         });
+
+        // Watch the store, change the fake history model when it changs
+        store.subscribe(({ type, payload: newId }) => {
+            if (type == "betaHistory/setCurrentHistoryId") {
+                console.log("setCurrentHistoryId", newId);
+                Galaxy.currHistoryPanel.setModel(new History({ id: newId }));
+            }
+        });
     },
     render() {
         // Hack: For now, remove unused "unified-panel" elements until we can

client/src/components/History/adapters/betaToggle.js

@@ -0,0 +1,13 @@
+export const isBetaHistoryOpen = () => {
+    return sessionStorage.getItem("useBetaHistory");
+};
+
+export const switchToLegacyHistoryPanel = () => {
+    sessionStorage.removeItem("useBetaHistory");
+    location.reload();
+};
+
+export const switchToBetaHistoryPanel = () => {
+    sessionStorage.setItem("useBetaHistory", 1);
+    location.reload(false);
+};

client/src/components/History/model/syncCurrentHistoryToGalaxy.js

@@ -1,9 +1,11 @@
 // Sync Galaxy store to legacy galaxy current history
 
-import { fromEvent } from "rxjs";
-import { map, filter, switchMap, pluck, distinctUntilChanged } from "rxjs/operators";
+import { concat, fromEvent, throwError } from "rxjs";
+import { tap, map, filter, switchMap, pluck, distinctUntilChanged, retryWhen, take, delay } from "rxjs/operators";
+
+export function syncCurrentHistoryToGalaxy(galaxy$, store, cfg = {}) {
+    const { retryPeriod = 500, retries = 20 } = cfg;
 
-export function syncCurrentHistoryToGalaxy(galaxy$, store) {
     // prettier-ignore
     const historyId$ = galaxy$.pipe(
         pluck("currHistoryPanel"),
@@ -14,15 +16,22 @@ export function syncCurrentHistoryToGalaxy(galaxy$, store) {
                 map(() => panel.model.id)
             );
         }),
+        retryWhen(err$ => {
+            const retries$ = err$.pipe(
+                delay(retryPeriod),
+                take(retries)
+            );
+            const errMsg$ = throwError("Unable to sync to currHistoryPanel because it never appeared");
+            return concat(retries$, errMsg$);
+        }),
         filter(Boolean),
-        distinctUntilChanged()
+        distinctUntilChanged(),
+        tap(() => console.log("Vuex is now tracking galaxy.currHistoryPanel")),
     );
 
     return historyId$.subscribe(
-        (id) => {
-            store.commit("betaHistory/setCurrentHistoryId", id);
-        },
-        (err) => console.log("syncCurrentHistoryToGalaxy error", err),
+        (id) => store.commit("betaHistory/setCurrentHistoryId", id),
+        (err) => console.warn("syncCurrentHistoryToGalaxy error", err),
         () => console.log("syncCurrentHistoryToGalaxy complete")
     );
 }

client/src/components/History/providers/HistoryContentProvider/loadContents.js

@@ -1,33 +1,44 @@
-import { pipe, defer, of } from "rxjs";
-import { repeat, switchMap, catchError } from "rxjs/operators";
-import { tag } from "rxjs-spy/operators/tag";
-import { decay } from "../../../../utils/observable/decay";
+import { defer, of } from "rxjs";
+import { repeat, switchMap, switchMapTo, startWith, debounceTime } from "rxjs/operators";
+import { decay } from "utils/observable/decay";
+import { monitorXHR } from "utils/observable/monitorXHR";
 import { loadHistoryContents } from "../../caching";
 
 // prettier-ignore
 export const loadContents = (cfg = {}) => {
     const {
         windowSize,
-        initialInterval = 3 * 1000,
+        initialInterval = 2 * 1000,
         maxInterval = 10 * initialInterval,
-        disablePoll = false,
+        disablePoll = false
     } = cfg;
 
-    return pipe(
-        switchMap(([{id}, params, hid]) => {
-            const singleLoad$ = defer(() => of([id, params, hid]).pipe(
-                tag('ajaxLoadInputs'),
-                loadHistoryContents({ windowSize }),
-            ));
-            const poll$ = singleLoad$.pipe(
-                decay({ initialInterval, maxInterval }),
-                repeat(),
-            );
-            return disablePoll ? singleLoad$ : poll$;
-        }),
-        catchError(err => {
-            console.warn("Error in loadContents", err);
-            throw err;
-        })
-    );
-}
+    return switchMap(([{ id }, params, hid]) => {
+
+        // a single history update
+        const singleLoad$ = of([id, params, hid]).pipe(
+            loadHistoryContents({ windowSize })
+        );
+
+        // start repeating, delay gets longer over time until unsubscribed
+        const freshPoll$ = defer(() => singleLoad$.pipe(
+            decay({ initialInterval, maxInterval }),
+            repeat()
+        ));
+
+        // history, tools routes all refresh
+        // exclude our own polling url though
+        const routes = [/api\/(history|tools|histories)/];
+        const methods = ["POST", "PUT", "DELETE"];
+        const resetPoll$ = monitorXHR({ methods, routes });
+
+        // resets re-subscribe to freshPoll$ starting the decay over again
+        const poll$ = resetPoll$.pipe(
+            startWith(true),
+            debounceTime(100),
+            switchMapTo(freshPoll$)
+        );
+
+        return disablePoll ? singleLoad$ : poll$;
+    });
+};

client/src/components/HistoryExport/ExportLink.vue

@@ -1,11 +1,12 @@
 <template>
     <span>
         <b
-            ><a :href="link">{{ link }}</a></b
+            ><a class="generated-export-link" :href="link">{{ link }}</a></b
         >
         <font-awesome-icon
             v-b-tooltip.hover
             title="Copy export URL to your clipboard"
+            class="copy-export-link"
             icon="link"
             style="cursor: pointer;"
             @click="copyUrl"
@@ -13,9 +14,9 @@
         <i
             title="Information about when the history export was generated is included in the job details. Additionally, if there are issues with export, the job details may help figure out the underlying problem or communicate issues to your Galaxy administrator."
         >
-            (<a href="#" @click="showDetails">view job details</a>)
+            (<a class="show-job-link" href="#" @click="showDetails">view job details</a>)
         </i>
-        <b-modal v-model="details" scrollable ok-only>
+        <b-modal v-model="details" modal-class="job-information-modal" scrollable ok-only hide-header>
             <job-information :job_id="historyExport.job_id" :include-times="true" />
         </b-modal>
     </span>

client/src/components/HistoryExport/Index.vue

@@ -1,5 +1,5 @@
 <template>
-    <span>
+    <span class="history-export-component">
         <h2>Export history archive</h2>
         <span v-if="initializing">
             <loading-span message="Loading server configuration." />
@@ -7,13 +7,15 @@
         <span v-else-if="hasWritableFileSources">
             <b-card no-body>
                 <b-tabs pills card vertical>
-                    <b-tab title="to a link" active>
+                    <b-tab title="to a link" title-link-class="tab-export-to-link" active>
                         <b-card-text>
                             <ToLink :history-id="historyId" />
                         </b-card-text>
                     </b-tab>
-                    <b-tab title="to a remote file">
-                        <ToRemoteFile :history-id="historyId" />
+                    <b-tab title="to a remote file" title-link-class="tab-export-to-file">
+                        <b-card-text>
+                            <ToRemoteFile :history-id="historyId" />
+                        </b-card-text>
                     </b-tab>
                 </b-tabs>
             </b-card>
@@ -25,20 +27,20 @@
 </template>
 
 <script>
-import Vue from "vue";
-import BootstrapVue from "bootstrap-vue";
+import { BCard, BTabs, BTab } from "bootstrap-vue";
 import ToLink from "./ToLink.vue";
 import ToRemoteFile from "./ToRemoteFile.vue";
 import { Services } from "components/FilesDialog/services";
 import LoadingSpan from "components/LoadingSpan";
 
-Vue.use(BootstrapVue);
-
 export default {
     components: {
         LoadingSpan,
         ToLink,
         ToRemoteFile,
+        BCard,
+        BTabs,
+        BTab,
     },
     props: {
         historyId: {
@@ -59,7 +61,7 @@ export default {
         async initialize() {
             const fileSources = await new Services().getFileSources();
             this.hasWritableFileSources = fileSources.some((fs) => fs.writable);
-            console.log(fileSources);
+            // console.log(fileSources);
             this.initializing = false;
         },
     },

client/src/components/HistoryExport/ToLink.vue

@@ -153,10 +153,7 @@ export default {
             waitOnJob(jobId)
                 .then((jobResponse) => {
                     this.waitingOnJob = false;
-                    this.loadingExports = true;
-                    // Race condition, for some reasons the job API returns
-                    // ok before the JEHA...
-                    setTimeout(this.loadExports, 2000);
+                    this.loadExports();
                 })
                 .catch(this.handleError);
         },

client/src/components/HistoryExport/ToRemoteFile.vue

@@ -1,5 +1,5 @@
 <template>
-    <div>
+    <div class="export-to-remote-file">
         <b-alert v-if="errorMessage" show variant="danger" dismissible @dismissed="errorMessage = null">
             {{ errorMessage }}
             <JobError
@@ -20,14 +20,14 @@
         </div>
         <div v-else>
             <b-form-group
-                id="directory"
+                id="fieldset-directory"
                 label-for="directory"
                 description="Select a 'remote files' directory to export history archive to."
                 class="mt-3"
             >
-                <files-input v-model="directory" mode="directory" :requireWritable="true" />
+                <files-input id="directory" v-model="directory" mode="directory" :requireWritable="true" />
             </b-form-group>
-            <b-form-group id="name" label-for="name" description="Give the exported file a name." class="mt-3">
+            <b-form-group id="fieldset-name" label-for="name" description="Give the exported file a name." class="mt-3">
                 <b-form-input id="name" v-model="name" placeholder="Name" required></b-form-input>
             </b-form-group>
             <b-row align-h="end">

client/src/components/HistoryImport.vue

@@ -1,5 +1,5 @@
 <template>
-    <b-card title="Import a history from an archive">
+    <b-card body-class="history-import-component" title="Import a history from an archive">
         <b-alert v-if="errorMessage" variant="danger" dismissible @dismissed="errorMessage = null" show>
             {{ errorMessage }}
             <JobError

client/src/components/JobInformation/JobError.vue

@@ -11,7 +11,7 @@
             </div>
             <!-- TODO: modal for reporting error. -->
         </b-card-text>
-        <b-modal v-model="showInfo" scrollable ok-only>
+        <b-modal v-model="showInfo" modal-class="job-information-modal" scrollable ok-only hide-header>
             <job-information :job_id="job.id" :include-times="true" />
         </b-modal>
     </b-card>

client/src/components/LibraryFolder/TopToolbar/FolderTopBar.vue

@@ -46,7 +46,7 @@
                                     from History</a
                                 >
                                 <a
-                                    v-if="user_library_import_dir"
+                                    v-if="user_library_import_dir_available"
                                     class="dropdown-item cursor-pointer"
                                     @click="addDatasets('userdir')"
                                 >
@@ -203,7 +203,7 @@ export default {
     data() {
         return {
             is_admin: false,
-            user_library_import_dir: false,
+            user_library_import_dir_available: false,
             library_import_dir: false,
             allow_library_path_paste: false,
             list_genomes: [],
@@ -225,7 +225,7 @@ export default {
         const Galaxy = getGalaxyInstance();
         this.services = new Services();
         this.is_admin = Galaxy.user.attributes.is_admin;
-        this.user_library_import_dir = Galaxy.config.user_library_import_dir;
+        this.user_library_import_dir_available = Galaxy.config.user_library_import_dir_available;
         this.library_import_dir = Galaxy.config.library_import_dir;
         this.allow_library_path_paste = Galaxy.config.allow_library_path_paste;
 

client/src/components/LibraryFolder/TopToolbar/add-datasets.js

@@ -411,6 +411,7 @@ var AddDatasets = Backbone.View.extend({
                 if (typeof response.responseJSON !== "undefined") {
                     if (response.responseJSON.err_code === 404001) {
                         Toast.warning(response.responseJSON.err_msg);
+                        getGalaxyInstance().modal.hide();
                     } else {
                         Toast.error(response.responseJSON.err_msg);
                     }

client/src/components/Panels/Buttons/FavoritesButton.vue

@@ -8,7 +8,8 @@
         role="button"
         aria-label="Show favorite tools"
     >
-        <font-awesome-icon :icon="['far', 'star']" />
+        <font-awesome-icon v-if="toggle" :icon="['fas', 'star']" />
+        <font-awesome-icon v-else :icon="['far', 'star']" />
     </a>
 </template>
 
@@ -17,22 +18,52 @@ import { VBTooltip } from "bootstrap-vue";
 import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
 import { library } from "@fortawesome/fontawesome-svg-core";
 import { faStar } from "@fortawesome/free-regular-svg-icons";
+import { faStar as faStarSolid } from "@fortawesome/free-solid-svg-icons";
+
 library.add(faStar);
+library.add(faStarSolid);
 
 export default {
     name: "FavoritesButton",
+    props: {
+        query: {
+            type: String,
+        },
+    },
     components: { FontAwesomeIcon },
     data() {
         return {
-            tooltipText: "Show favorites",
+            searchKey: "#favorites",
+            tooltipToggle: "Show favorites",
+            tooltipUntoggle: "Clear",
+            toggle: false,
         };
     },
     directives: {
         "v-b-tooltip": VBTooltip,
     },
+    watch: {
+        query() {
+            this.toggle = this.query == this.searchKey;
+        },
+    },
+    computed: {
+        tooltipText() {
+            if (this.toggle) {
+                return this.tooltipUntoggle;
+            } else {
+                return this.tooltipToggle;
+            }
+        },
+    },
     methods: {
         onFavorites() {
-            this.$emit("onFavorites", "#favorites");
+            this.toggle = !this.toggle;
+            if (this.toggle) {
+                this.$emit("onFavorites", this.searchKey);
+            } else {
+                this.$emit("onFavorites", null);
+            }
         },
     },
 };

client/src/components/Panels/ToolBox.vue

@@ -3,7 +3,7 @@
         <div class="unified-panel-header" unselectable="on">
             <div class="unified-panel-header-inner">
                 <div class="panel-header-buttons">
-                    <favorites-button @onFavorites="onFavorites" v-if="isUser" />
+                    <favorites-button :query="query" @onFavorites="onFavorites" v-if="isUser" />
                 </div>
                 <div class="panel-header-text">Tools</div>
             </div>

client/src/components/Panels/utilities.js

@@ -1,10 +1,14 @@
 export function filterToolSections(layout, results) {
     if (results) {
+        results = normalize_results(results);
         const filteredLayout = layout.map((section) => {
             var toolRes = [];
             if (section.elems) {
                 section.elems.forEach((el) => {
-                    if (!el.text && results.includes(el.id)) {
+                    if (
+                        (!el.text && results.includes(el.id)) ||
+                        (el.tool_shed_repository && results.includes(el.tool_shed_repository.name))
+                    ) {
                         toolRes.push(el);
                     }
                 });
@@ -41,6 +45,7 @@ export function filterTools(layout, results) {
     // do not consider expression tools, unless requested by the workflow editor
     layout = layout.filter((section) => section.name !== "Expression Tools");
     if (results) {
+        results = normalize_results(results);
         var toolsResults = [];
         if (results.length < 1) {
             return toolsResults;
@@ -50,7 +55,10 @@ export function filterTools(layout, results) {
         layout.map((section) => {
             if (section.elems) {
                 section.elems.forEach((el) => {
-                    if (!el.text && results.includes(el.id)) {
+                    if (
+                        (!el.text && results.includes(el.id)) ||
+                        (el.tool_shed_repository && results.includes(el.tool_shed_repository.name))
+                    ) {
                         toolsResults.push(el);
                     }
                 });
@@ -65,3 +73,15 @@ export function filterTools(layout, results) {
         return layout;
     }
 }
+
+function normalize_results(results) {
+    var norm_results = [];
+    results.forEach((result) => {
+        if (result.includes("/repos/")) {
+            norm_results.push(result.split("/")[3]);
+        } else {
+            norm_results.push(result);
+        }
+    });
+    return norm_results;
+}

client/src/components/ToolsView/ToolsSchemaJson/ToolsJson.test.js

@@ -3,7 +3,10 @@ import ToolsJson from "./ToolsJson";
 import testToolsListResponse from "../testData/toolsList";
 import MockAdapter from "axios-mock-adapter";
 import axios from "axios";
-import { mount } from "@vue/test-utils";
+import { shallowMount } from "@vue/test-utils";
+import { getLocalVue } from "jest/helpers";
+
+const localVue = getLocalVue();
 
 describe("ToolsView/ToolsView.vue", () => {
     let wrapper;
@@ -12,7 +15,7 @@ describe("ToolsView/ToolsView.vue", () => {
 
     beforeEach(async () => {
         axiosMock = new MockAdapter(axios);
-        wrapper = mount(ToolsJson);
+        wrapper = shallowMount(ToolsJson, { localVue });
         axiosMock.onGet("/api/tools?tool_help=True").reply(200, testToolsListResponse);
         await wrapper.vm.$nextTick();
     });

client/src/components/Toolshed/InstalledList/Index.test.js

@@ -1,4 +1,5 @@
 import { mount } from "@vue/test-utils";
+import { getLocalVue } from "jest/helpers";
 import Index from "./Index";
 
 jest.mock("app");
@@ -10,6 +11,8 @@ getAppRoot.mockImplementation(() => "/");
 import { Services } from "../services";
 jest.mock("../services");
 
+const localVue = getLocalVue();
+
 Services.mockImplementation(() => {
     return {
         async getInstalledRepositories() {
@@ -42,6 +45,7 @@ describe("InstalledList", () => {
             stubs: {
                 RepositoryDetails: true,
             },
+            localVue,
         });
         expect(wrapper.find(".loading-message").text()).toBe("Loading installed repositories...");
         await wrapper.vm.$nextTick();

client/src/components/Toolshed/RepositoryDetails/InstallationButton.test.js

@@ -1,6 +1,9 @@
 import { mount } from "@vue/test-utils";
+import { getLocalVue } from "jest/helpers";
 import InstallationButton from "./InstallationButton";
 
+const localVue = getLocalVue();
+
 describe("InstallationButton", () => {
     it("test installed repository revision", () => {
         const wrapper = mount(InstallationButton, {
@@ -8,6 +11,7 @@ describe("InstallationButton", () => {
                 installed: true,
                 status: "Installed",
             },
+            localVue,
         });
         const $el = wrapper.find("button");
         expect($el.classes()).toEqual(expect.arrayContaining(["btn-danger"]));

client/src/components/Toolshed/RepositoryDetails/InstallationSettings.test.js

@@ -1,8 +1,11 @@
 import { mount } from "@vue/test-utils";
+import { getLocalVue } from "jest/helpers";
 import InstallationSettings from "./InstallationSettings";
 
 jest.mock("app");
 
+const localVue = getLocalVue();
+
 describe("InstallationSettings", () => {
     it("test tool repository installer interface", () => {
         const wrapper = mount(InstallationSettings, {
@@ -18,6 +21,7 @@ describe("InstallationSettings", () => {
                 requiresPanel: true,
                 toolshedUrl: "toolshedUrl",
             },
+            localVue,
         });
         expect(wrapper.find(".title").text()).toBe("Installing 'name'");
         expect(wrapper.find(".description").text()).toBe("long_description");

client/src/components/Toolshed/RepositoryDetails/RepositoryTools.test.js

@@ -1,6 +1,9 @@
 import { mount } from "@vue/test-utils";
+import { getLocalVue } from "jest/helpers";
 import RepositoryTools from "./RepositoryTools";
 
+const localVue = getLocalVue();
+
 describe("RepositoryTools", () => {
     it("test tool version list in repository details", () => {
         const wrapper = mount(RepositoryTools, {
@@ -13,6 +16,7 @@ describe("RepositoryTools", () => {
                     },
                 ],
             },
+            localVue,
         });
         const $el = wrapper.findAll("td");
         const $first = $el.at(0);

client/src/components/Toolshed/SearchList/ServerSelection.test.js

@@ -1,6 +1,9 @@
 import { mount } from "@vue/test-utils";
+import { getLocalVue } from "jest/helpers";
 import ServerSelection from "./ServerSelection";
 
+const localVue = getLocalVue();
+
 describe("ServerSelection", () => {
     it("test server selection dropdown", () => {
         const wrapper = mount(ServerSelection, {
@@ -10,6 +13,7 @@ describe("ServerSelection", () => {
                 loading: false,
                 total: "total",
             },
+            localVue,
         });
         expect(wrapper.find(".description").text()).toBe("total repositories available at");
         const $options = wrapper.findAll("a");

client/src/components/Upload/UploadModal.test.js

@@ -3,7 +3,6 @@ import axios from "axios";
 import UploadModal from "./UploadModal.vue";
 import store from "../../store";
 import { shallowMount, createLocalVue } from "@vue/test-utils";
-import BootstrapVue from "bootstrap-vue";
 
 jest.mock("app");
 jest.mock("../History/caching");
@@ -39,7 +38,6 @@ describe("UploadModal.vue", () => {
         axiosMock.onGet(`/api/genomes`).reply(200, genomesResponse);
 
         const localVue = createLocalVue();
-        localVue.use(BootstrapVue);
 
         wrapper = await shallowMount(UploadModal, {
             store,

client/src/components/Upload/UploadModal.vue

@@ -46,6 +46,7 @@ import RulesInput from "./RulesInput";
 import LoadingSpan from "components/LoadingSpan";
 import { mapState } from "vuex";
 import { BModal, BTabs, BTab } from "bootstrap-vue";
+import { isBetaHistoryOpen } from "components/History/adapters/betaToggle";
 
 const UploadModal = {
     components: {
@@ -336,8 +337,7 @@ const UploadModal = {
 };
 
 // Beta history patch
-const useBetaHistory = sessionStorage.getItem("useBetaHistory");
-if (useBetaHistory) {
+if (isBetaHistoryOpen()) {
     UploadModal.computed.currentHistoryId = function () {
         return this.$store.getters["betaHistory/currentHistoryId"];
     };

client/src/components/User/CloudAuth/CloudAuth.test.js

@@ -5,9 +5,6 @@ import { getLocalVue } from "jest/helpers";
 import { default as CloudAuth } from "./CloudAuth";
 import CloudAuthItem from "./CloudAuthItem";
 
-import _l from "utils/localization";
-import BootstrapVue from "bootstrap-vue";
-
 jest.mock("./model/service", () => ({
     listCredentials: async () => {
         const listCredentials = require("./testdata/listCredentials.json");
@@ -17,8 +14,6 @@ jest.mock("./model/service", () => ({
 }));
 
 const localVue = getLocalVue();
-localVue.use(BootstrapVue);
-localVue.filter("localize", (value) => _l(value));
 
 describe("CloudAuth component", () => {
     let wrapper;

client/src/components/User/RecentInvocations.test.js

@@ -1,5 +1,6 @@
 import Invocations from "../Workflow/Invocations";
 import { mount } from "@vue/test-utils";
+import { getLocalVue } from "jest/helpers";
 import flushPromises from "flush-promises";
 
 import RecentInvocations from "./RecentInvocations.vue";
@@ -7,13 +8,15 @@ import RecentInvocations from "./RecentInvocations.vue";
 jest.mock("../History/caching");
 jest.mock("./UserServices");
 
+const localVue = getLocalVue();
+
 describe("RecentInvocations.vue", () => {
     let wrapper;
     let fetchInvocationsSpy;
 
     beforeEach(async () => {
         fetchInvocationsSpy = jest.spyOn(RecentInvocations.methods, "fetchRecentInvocations");
-        wrapper = mount(RecentInvocations);
+        wrapper = mount(RecentInvocations, { localVue });
         await flushPromises();
     });
 

client/src/components/Workflow/Editor/RefactorConfirmationModal.test.js

@@ -1,11 +1,12 @@
 jest.mock("./modules/services");
 
-import { shallowMount, createLocalVue } from "@vue/test-utils";
+import { shallowMount } from "@vue/test-utils";
+import { getLocalVue } from "jest/helpers";
 import RefactorConfirmationModal from "./RefactorConfirmationModal";
 import { refactor } from "./modules/services";
 import flushPromises from "flush-promises";
 
-const localVue = createLocalVue();
+const localVue = getLocalVue();
 const TEST_WORKFLOW_ID = "test123";
 const TEST_ACTION_TYPE = "upgrade_subworkflow";
 

client/src/components/Workflow/Invocations.test.js

@@ -1,10 +1,13 @@
 import Invocations from "../Workflow/Invocations";
 import { mount } from "@vue/test-utils";
+import { getLocalVue } from "jest/helpers";
 import invocationData from "./test/json/invocation.json";
 import moment from "moment";
 
 jest.mock("../History/caching");
 
+const localVue = getLocalVue();
+
 describe("Invocations.vue without invocation", () => {
     let wrapper;
     let propsData;
@@ -16,6 +19,7 @@ describe("Invocations.vue without invocation", () => {
         };
         wrapper = mount(Invocations, {
             propsData,
+            localVue,
         });
     });
 
@@ -62,6 +66,7 @@ describe("Invocations.vue with invocation", () => {
                     template: "<span/>",
                 },
             },
+            localVue,
         });
     });
 
@@ -81,10 +86,10 @@ describe("Invocations.vue with invocation", () => {
     it("toggles detail rendering", async () => {
         let rows = wrapper.findAll("tbody > tr").wrappers;
         expect(rows.length).toBe(1);
-        await wrapper.find("#toggle-invocation-details").trigger("click");
+        await wrapper.find(".toggle-invocation-details").trigger("click");
         rows = wrapper.findAll("tbody > tr").wrappers;
         expect(rows.length).toBe(3);
-        await wrapper.find("#toggle-invocation-details").trigger("click");
+        await wrapper.find(".toggle-invocation-details").trigger("click");
         rows = wrapper.findAll("tbody > tr").wrappers;
         expect(rows.length).toBe(1);
     });

client/src/components/Workflow/Invocations.vue

@@ -1,5 +1,5 @@
 <template>
-    <div>
+    <div class="invocations-list">
         <h2 class="mb-3">
             <span id="invocations-title">Workflow Invocations</span>
         </h2>
@@ -24,7 +24,7 @@
             >
                 <template v-slot:row-details="row">
                     <b-card>
-                        <small class="float-right">
+                        <small class="float-right" :data-invocation-id="row.item.id">
                             <b>Invocation: {{ row.item.id }}</b>
                         </small>
                         <workflow-invocation-state
@@ -35,7 +35,7 @@
                 </template>
                 <template v-slot:cell(workflow_id)="data">
                     <b-link
-                        id="toggle-invocation-details"
+                        class="toggle-invocation-details"
                         v-b-tooltip
                         title="Show Invocation details"
                         href="#"

client/src/components/WorkflowInvocationState/JobStep.test.js

@@ -0,0 +1,71 @@
+import BootstrapVue from "bootstrap-vue";
+import JobStep from "./JobStep";
+import { mount } from "@vue/test-utils";
+import jobs from "./test/json/jobs.json";
+
+jest.mock("../History/caching");
+
+import { createLocalVue } from "@vue/test-utils";
+
+// create an extended `Vue` constructor
+const localVue = createLocalVue();
+
+// install plugins as normal
+localVue.use(BootstrapVue);
+
+describe("DatasetUIWrapper.vue with Dataset", () => {
+    let wrapper;
+    let propsData;
+
+    beforeEach(async () => {
+        propsData = {
+            jobs: jobs,
+        };
+        wrapper = mount(JobStep, {
+            localVue,
+            propsData,
+            stubs: {
+                JobProvider: { template: "<div class='expanded'>I am expanded</div>" },
+            },
+        });
+    });
+    test("it renders a table with 2 jobs", async () => {
+        expect(wrapper.find("tbody").findAll("tr").length).toBe(2);
+    });
+    test("it expands on row click", async () => {
+        // verify no item is expanded
+        expect(wrapper.vm.toggledItems["1"]).toBe(undefined);
+        expect(wrapper.find(".expanded").exists()).toBeFalsy();
+        // expand
+        wrapper.find("tbody").find("tbody").find("tr").trigger("click");
+        await localVue.nextTick();
+        expect(wrapper.vm.toggledItems["1"]).toBeTruthy();
+        expect(wrapper.find(".expanded").exists()).toBeTruthy();
+        wrapper.find("tbody").find("tbody").find("tr").trigger("click");
+        // close again
+        await localVue.nextTick();
+        expect(wrapper.vm.toggledItems["1"]).toBeFalsy();
+        expect(wrapper.find(".expanded").exists()).toBeFalsy();
+    });
+    test("it sustains expanded on prop update", async () => {
+        // verify no item is expanded
+        expect(wrapper.vm.toggledItems["1"]).toBe(undefined);
+        expect(wrapper.find(".expanded").exists()).toBeFalsy();
+        // expand
+        wrapper.find("tbody").find("tbody").find("tr").trigger("click");
+        await localVue.nextTick();
+        expect(wrapper.vm.toggledItems["1"]).toBeTruthy();
+        expect(wrapper.find(".expanded").exists()).toBeTruthy();
+        // 2 collapsed rows, plus 1 expanded row
+        expect(wrapper.find("tbody").findAll("tr").length).toBe(3);
+        // update data
+        const additionalJob = { ...jobs[0], id: 3 };
+        wrapper.setProps({ jobs: [...jobs, additionalJob] });
+        await localVue.nextTick();
+        // verify new data is displayed
+        expect(wrapper.find("tbody").findAll("tr").length).toBe(4);
+        // verify first row is still expanded
+        expect(wrapper.vm.toggledItems["1"]).toBeTruthy();
+        expect(wrapper.find(".expanded").exists()).toBeTruthy();
+    });
+});

client/src/components/WorkflowInvocationState/JobStep.vue

@@ -1,13 +1,6 @@
 <template>
     <b-card v-if="jobs">
-        <b-table
-            small
-            caption-top
-            :items="jobs"
-            :fields="fields"
-            v-if="jobs"
-            @row-clicked="(item) => $set(item, '_showDetails', !item._showDetails)"
-        >
+        <b-table small caption-top :items="jobsProvider" :fields="fields" primary-key="id" @row-clicked="toggleDetails">
             <template v-slot:row-details="row">
                 <job-provider
                     :id="row.item.id"
@@ -53,6 +46,31 @@ export default {
     props: {
         jobs: { type: Array, required: true },
     },
+    methods: {
+        jobsProvider(ctx, callback) {
+            // It may seem unnecessary to use a provider here, since the jobs prop
+            // is being updated externally. However we need to keep track of the
+            // _showDetails attribute which determines whether the row is shown as expanded
+            this.$watch(
+                "jobs",
+                function () {
+                    // update new jobs array with current state
+                    const toggledJobs = this.jobs.map((e) => {
+                        return { ...e, _showDetails: !!this.toggledItems[e.id] };
+                    });
+                    callback(toggledJobs);
+                },
+                { immediate: true }
+            );
+            return null;
+        },
+        toggleDetails(item) {
+            // toggle item
+            item._showDetails = !item._showDetails;
+            // update state
+            this.toggledItems[item.id] = item._showDetails;
+        },
+    },
     data() {
         return {
             fields: [
@@ -60,6 +78,7 @@ export default {
                 { key: "update_time", label: "Updated", sortable: true },
                 { key: "create_time", label: "Created", sortable: true },
             ],
+            toggledItems: {},
         };
     },
 };

client/src/components/WorkflowInvocationState/WorkflowInvocationDetails.vue

@@ -1,22 +1,26 @@
 <template>
     <div v-if="invocation">
         <div v-if="Object.keys(invocation.input_step_parameters).length">
-            <details
+            <details class="invocation-parameters-details"
                 ><summary><b>Parameters</b></summary>
                 <parameter-step :parameters="Object.values(invocation.input_step_parameters)" />
             </details>
         </div>
         <div v-if="Object.keys(invocation.inputs).length">
-            <details
+            <details class="invocation-inputs-details"
                 ><summary><b>Inputs</b></summary>
-                <div v-for="(input, key) in invocation.inputs" :key="input.id">
+                <div
+                    v-for="(input, key) in invocation.inputs"
+                    :key="input.id"
+                    :data-label="dataInputStepLabel(key, input)"
+                >
                     <b>{{ dataInputStepLabel(key, input) }}</b>
                     <workflow-invocation-data-contents :data_item="input" />
                 </div>
             </details>
         </div>
         <div v-if="Object.keys(invocation.outputs).length">
-            <details
+            <details class="invocation-outputs-details"
                 ><summary><b>Outputs</b></summary>
                 <div v-for="(output, key) in invocation.outputs" :key="output.id">
                     <b>{{ key }}:</b>
@@ -25,7 +29,7 @@
             </details>
         </div>
         <div v-if="Object.keys(invocation.output_collections).length">
-            <details
+            <details class="invocation-output-collections-details"
                 ><summary><b>Output Collections</b></summary>
                 <div v-for="(output, key) in invocation.output_collections" :key="output.id">
                     <b>{{ key }}:</b>
@@ -34,7 +38,7 @@
             </details>
         </div>
         <div v-if="workflow">
-            <details v-if="workflow"
+            <details v-if="workflow" class="invocation-steps-details"
                 ><summary><b>Steps</b></summary>
                 <workflow-invocation-step
                     v-for="step in Object.values(workflow.steps)"

client/src/components/WorkflowInvocationState/WorkflowInvocationState.test.js

@@ -1,5 +1,6 @@
 import WorkflowInvocationState from "./WorkflowInvocationState";
 import { shallowMount } from "@vue/test-utils";
+import { getLocalVue } from "jest/helpers";
 import Vuex from "vuex";
 import invocationData from "../Workflow/test/json/invocation.json";
 
@@ -11,6 +12,8 @@ const invocationJobsSummaryById = {
 };
 jest.mock("../History/caching");
 
+const localVue = getLocalVue();
+
 describe("WorkflowInvocationState.vue with terminal invocation", () => {
     let wrapper;
     let propsData;
@@ -25,6 +28,7 @@ describe("WorkflowInvocationState.vue with terminal invocation", () => {
                 invocation: () => invocationData,
                 getInvocationJobsSummaryById: () => () => invocationJobsSummaryById,
             },
+            localVue,
         });
     });
 
@@ -33,13 +37,13 @@ describe("WorkflowInvocationState.vue with terminal invocation", () => {
     });
 
     it("displays report links", async () => {
-        expect(wrapper.find("#invocation-pdf-link").exists()).toBeTruthy();
-        expect(wrapper.find("#invocation-link").exists()).toBeTruthy();
-        expect(wrapper.find("#bco-json").exists()).toBeTruthy();
+        expect(wrapper.find(".invocation-pdf-link").exists()).toBeTruthy();
+        expect(wrapper.find(".invocation-report-link").exists()).toBeTruthy();
+        expect(wrapper.find(".bco-json").exists()).toBeTruthy();
     });
 
     it("doesn't show cancel invocation button", async () => {
-        expect(wrapper.find("#cancel-workflow-scheduling").exists()).toBeFalsy();
+        expect(wrapper.find(".cancel-workflow-scheduling").exists()).toBeFalsy();
     });
 });
 
@@ -67,6 +71,7 @@ describe("WorkflowInvocationState.vue with no invocation", () => {
                 invocation: () => null,
                 getInvocationJobsSummaryById: () => () => invocationJobsSummaryById,
             },
+            localVue,
         });
     });
 
@@ -75,12 +80,12 @@ describe("WorkflowInvocationState.vue with no invocation", () => {
     });
 
     it("does not display report links", async () => {
-        expect(wrapper.find("#invocation-pdf-link").exists()).toBeFalsy();
-        expect(wrapper.find("#invocation-link").exists()).toBeFalsy();
-        expect(wrapper.find("#bco-json").exists()).toBeFalsy();
+        expect(wrapper.find(".invocation-pdf-link").exists()).toBeFalsy();
+        expect(wrapper.find(".invocation-report-link").exists()).toBeFalsy();
+        expect(wrapper.find(".bco-json").exists()).toBeFalsy();
     });
 
     it("shows cancel invocation button", async () => {
-        expect(wrapper.find("#cancel-workflow-scheduling").exists()).toBeTruthy();
+        expect(wrapper.find(".cancel-workflow-scheduling").exists()).toBeTruthy();
     });
 });

client/src/components/WorkflowInvocationState/WorkflowInvocationState.vue

@@ -1,13 +1,12 @@
 <template>
-    <div class="mb-3">
+    <div class="mb-3 workflow-invocation-state-component">
         <div v-if="invocationAndJobTerminal">
             <span>
-                <a id="invocation-link" :href="invocationLink"
-                    ><b>View Report {{ index + 1 }}</b></a
+                <a class="invocation-report-link" :href="invocationLink"
+                    ><b>View Report {{ indexStr }}</b></a
                 >
                 <a
-                    id="invocation-pdf-link"
-                    class="fa fa-print ml-1"
+                    class="fa fa-print ml-1 invocation-pdf-link"
                     :href="invocationPdfLink"
                     v-b-tooltip
                     title="Download PDF"
@@ -16,26 +15,27 @@
         </div>
         <div v-else>
             <span class="fa fa-spinner fa-spin" />
-            <span>Invocation {{ index + 1 }}...</span>
+            <span>Invocation {{ indexStr }}...</span>
             <span
-                id="cancel-workflow-scheduling"
+                class="fa fa-times cancel-workflow-scheduling"
                 v-if="!invocationSchedulingTerminal"
                 v-b-tooltip.hover
                 title="Cancel scheduling of workflow invocation"
-                class="fa fa-times"
                 @click="cancelWorkflowScheduling"
             ></span>
         </div>
-        <progress-bar v-if="!stepCount" note="Loading step state summary..." :loading="true" />
+        <progress-bar v-if="!stepCount" note="Loading step state summary..." :loading="true" class="steps-progress" />
         <progress-bar
             v-else-if="invocationState == 'cancelled'"
             note="Invocation scheduling cancelled - expected jobs and outputs may not be generated."
             :error-count="1"
+            class="steps-progress"
         />
         <progress-bar
             v-else-if="invocationState == 'failed'"
             note="Invocation scheduling failed - Galaxy administrator may have additional details in logs."
             :error-count="1"
+            class="steps-progress"
         />
         <progress-bar
             v-else
@@ -43,6 +43,7 @@
             :total="stepCount"
             :ok-count="stepStates.scheduled"
             :loading="!invocationSchedulingTerminal"
+            class="steps-progress"
         />
         <progress-bar
             :note="jobStatesStr"
@@ -52,9 +53,10 @@
             :new-count="newCount"
             :error-count="errorCount"
             :loading="!invocationAndJobTerminal"
+            class="jobs-progress"
         />
         <span v-if="invocationAndJobTerminal">
-            <a id="bco-json" :href="bcoJSON"><b>Download BioCompute Object</b></a>
+            <a class="bco-json" :href="bcoJSON"><b>Download BioCompute Object</b></a>
         </span>
         <workflow-invocation-details
             v-if="invocation"
@@ -89,7 +91,7 @@ export default {
         },
         index: {
             type: Number,
-            default: 0,
+            optional: true,
         },
     },
     data() {
@@ -104,6 +106,13 @@ export default {
     },
     computed: {
         ...mapGetters(["getInvocationById", "getInvocationJobsSummaryById"]),
+        indexStr() {
+            if (this.index == null) {
+                return "";
+            } else {
+                return `${this.index + 1}`;
+            }
+        },
         invocation: function () {
             return this.getInvocationById(this.invocationId);
         },

client/src/components/WorkflowInvocationState/WorkflowInvocationStep.vue

@@ -1,10 +1,10 @@
 <template>
-    <div class="d-flex">
+    <div class="d-flex" :data-step="workflowStep.id">
         <div class="ui-portlet-section" style="width: 100%;">
             <div class="portlet-header portlet-title portlet-operations" v-on:click="toggleStep">
                 <i :class="'portlet-title-icon fa mr-1 ' + stepIcon"></i>
                 <span class="portlet-title-text">
-                    <u>{{ stepLabel }}</u>
+                    <u class="step-title">{{ stepLabel }}</u>
                 </span>
             </div>
             <div class="portlet-content" v-if="expanded">
@@ -16,14 +16,18 @@
                     <div style="min-width: 1;">
                         <loading-span v-if="loading" :message="`Loading invocation step details`"> </loading-span>
                         <div v-else>
-                            <details v-if="Object.values(stepDetails.outputs).length > 0"
+                            <details
+                                v-if="Object.values(stepDetails.outputs).length > 0"
+                                class="invocation-step-output-details"
                                 ><summary><b>Output Datasets</b></summary>
                                 <div v-for="(value, name) in stepDetails.outputs" :key="value.id">
                                     <b>{{ name }}</b>
                                     <workflow-invocation-data-contents :data_item="value" />
                                 </div>
                             </details>
-                            <details v-if="Object.values(stepDetails.output_collections).length > 0"
+                            <details
+                                v-if="Object.values(stepDetails.output_collections).length > 0"
+                                class="invocation-step-output-collection-details"
                                 ><summary><b>Output Dataset Collections</b></summary>
                                 <div v-for="(value, name) in stepDetails.output_collections" :key="value.id">
                                     <b>{{ name }}</b>
@@ -31,7 +35,7 @@
                                 </div>
                             </details>
                             <div class="portlet-body" style="width: 100%; overflow-x: auto;">
-                                <details v-if="workflowStepType == 'tool'">
+                                <details v-if="workflowStepType == 'tool'" class="invocation-step-job-details">
                                     <summary><b>Jobs</b></summary>
                                     <job-step :jobs="stepDetails.jobs" />
                                 </details>

client/src/components/WorkflowInvocationState/test/json/jobs.json

@@ -0,0 +1,24 @@
+[
+    {
+        "model_class": "Job",
+        "id": "1",
+        "state": "new",
+        "exit_code": 0,
+        "update_time": "2021-01-15T10:38:36.015269",
+        "create_time": "2021-01-15T10:38:21.462106",
+        "galaxy_version": "21.01",
+        "tool_id": "cat_data_and_sleep",
+        "history_id": "b6aaf20167e56df2"
+    },
+    {
+        "model_class": "Job",
+        "id": "2",
+        "state": "new",
+        "exit_code": 0,
+        "update_time": "2021-01-15T10:38:36.015269",
+        "create_time": "2021-01-15T10:38:21.462106",
+        "galaxy_version": "21.01",
+        "tool_id": "cat_data_and_sleep",
+        "history_id": "b6aaf20167e56df2"
+    }
+]

client/src/components/admin/BaseList.test.js

@@ -1,6 +1,9 @@
 import { mount } from "@vue/test-utils";
+import { getLocalVue } from "jest/helpers";
 import BaseList from "./BaseList";
 
+const localVue = getLocalVue();
+
 describe("Categories", () => {
     const getter = async () => {
         return {
@@ -30,6 +33,7 @@ describe("Categories", () => {
                 getter: getter,
                 setter: setter,
             },
+            localVue,
         });
         await wrapper.vm.$nextTick();
         expect(wrapper.find(".card-header").text()).toContain("There are 2");

client/src/entry/analysis/index.js

@@ -6,14 +6,14 @@ import Page from "layout/page";
 
 // Vue adapter emulates current features of backbone history panel
 import { HistoryPanelProxy } from "components/History";
+import { isBetaHistoryOpen } from "components/History/adapters/betaToggle";
 
 addInitialization((Galaxy, { options = {} }) => {
     console.log("Analysis custom page setup");
 
     // Handle beta history panel
     // Need to mock Galaxy.currHistoryPanel
-    const useBeta = sessionStorage.getItem("useBetaHistory");
-    const HistoryPanel = useBeta ? HistoryPanelProxy : MvcHistoryPanel;
+    const HistoryPanel = isBetaHistoryOpen() ? HistoryPanelProxy : MvcHistoryPanel;
 
     const pageOptions = Object.assign({}, options, {
         config: Object.assign({}, options.config, {

client/src/mvc/history/options-menu.js

@@ -6,6 +6,7 @@ import _l from "utils/localization";
 import PopupMenu from "mvc/ui/popup-menu";
 import historyCopyDialog from "mvc/history/copy-dialog";
 import Webhooks from "mvc/webhooks";
+import { switchToBetaHistoryPanel } from "../../components/History/adapters/betaToggle";
 
 // ============================================================================
 var menu = [
@@ -165,8 +166,7 @@ var menu = [
         html: _l("Use Beta History Panel"),
         anon: false,
         func: function () {
-            sessionStorage.setItem("useBetaHistory", 1);
-            window.location.reload(false);
+            switchToBetaHistoryPanel();
         },
     },
 ];

client/src/store/syncVuextoGalaxy.js

@@ -13,6 +13,7 @@ import { waitForInit } from "utils/observable/waitForInit";
 import { syncUserToGalaxy } from "store/userStore/syncUserToGalaxy";
 import { syncConfigToGalaxy } from "store/configStore/syncConfigToGalaxy";
 import { syncCurrentHistoryToGalaxy } from "components/History/model/syncCurrentHistoryToGalaxy";
+import { isBetaHistoryOpen } from "components/History/adapters/betaToggle";
 
 export const syncVuextoGalaxy = (store) => {
     const globalGalaxy$ = defer(() => {
@@ -25,6 +26,8 @@ export const syncVuextoGalaxy = (store) => {
     // configuration
     syncConfigToGalaxy(globalGalaxy$, store);
 
-    // Current History object (from legacy history panel)
-    syncCurrentHistoryToGalaxy(globalGalaxy$, store);
+    // Update Vuex with legacy galaxy history changes only when beta panel is closed.
+    if (!isBetaHistoryOpen()) {
+        syncCurrentHistoryToGalaxy(globalGalaxy$, store);
+    }
 };

client/src/utils/observable/decay.js

@@ -1,14 +1,9 @@
 // Stateful delay gets longer with each repeated call
-import { pipe, of } from "rxjs";
+import { of } from "rxjs";
 import { mergeMap, delay } from "rxjs/operators";
 
-// prettier-ignore
 export const decay = (cfg = {}) => {
-    const {
-        initialInterval,
-        maxInterval = 60 * 1000,
-        lambda = 0.25
-    } = cfg;
+    const { initialInterval, maxInterval = 60 * 1000, lambda = 0.25 } = cfg;
 
     if (undefined === initialInterval) {
         throw new Error("provide an initialInterval to decay");
@@ -16,13 +11,10 @@ export const decay = (cfg = {}) => {
 
     let counter = 0;
 
-    return pipe(
-        mergeMap(val => {
-            let waitTime = Math.floor(initialInterval * Math.exp(lambda * counter++));
-            waitTime = Math.max(waitTime, initialInterval);
-            waitTime = Math.min(waitTime, maxInterval);
-            // console.log("decay time", waitTime);
-            return of(val).pipe(delay(waitTime))
-        })
-    );
-}
+    return mergeMap((val) => {
+        let waitTime = Math.floor(initialInterval * Math.exp(lambda * counter++));
+        waitTime = Math.max(waitTime, initialInterval);
+        waitTime = Math.min(waitTime, maxInterval);
+        return of(val).pipe(delay(waitTime));
+    });
+};

client/src/utils/observable/monitorXHR.js

@@ -0,0 +1,62 @@
+import { defer, Subject } from "rxjs";
+import { filter, finalize, share } from "rxjs/operators";
+
+// global XHR request feed, emits every time xhr fires
+const xhr$ = defer(() => {
+    const request$ = new Subject();
+
+    // patch
+    const originalOpen = XMLHttpRequest.prototype.open;
+    XMLHttpRequest.prototype.open = function (method, url) {
+        request$.next({ method, url });
+        return originalOpen.apply(this, arguments);
+    };
+
+    return request$.pipe(
+        // un-patch after all subscribers have left
+        finalize(() => {
+            XMLHttpRequest.prototype.open = originalOpen;
+        })
+    );
+}).pipe(share());
+
+/**
+ * Watch outgoing ajax calls for any of the indicated methods, filter to any of the provided route regexes
+ *
+ * @param {Object} cfg  Configure methods and routes to monitor
+ * @return {Observable} Observable that emits every time a matching outgoing request happens
+ */
+export const monitorXHR = (cfg = {}) => {
+    return xhr$.pipe(filter(matchRoutes(cfg)));
+};
+
+// matches incoming method/url against config
+export const matchRoutes = (cfg = {}) => ({ method, url }) => {
+    const { methods = ["GET", "POST", "PUT", "DELETE"], routes = [], exclude = [] } = cfg;
+
+    // match fun compares route def to current url
+    const matcher = matchUrlToRoute(url);
+
+    if (!methods.includes(method)) {
+        return false;
+    }
+    if (exclude.length && exclude.find(matcher)) {
+        return false;
+    }
+    if (routes.length) {
+        const routeMatch = routes.find(matcher);
+        return !!routeMatch;
+    }
+    return false;
+};
+
+const matchUrlToRoute = (url) => (rt) => {
+    let result = false;
+    if (rt instanceof RegExp) {
+        result = url.match(rt);
+    } else {
+        // simple matcher
+        result = url.includes(rt);
+    }
+    return result;
+};

client/src/utils/observable/monitorXHR.test.js

@@ -0,0 +1,87 @@
+import { matchRoutes, monitorXHR } from "./monitorXHR";
+
+describe("monitorXhr", () => {
+    describe("monitorXHR", () => {
+        it("should import", () => {
+            expect(monitorXHR).toBeInstanceOf(Function);
+        });
+    });
+
+    // request matcher, XHR object emits objects of form { method, url } which get matched to a set
+    // of routes of interest
+
+    describe("matchRoutes", () => {
+        let matcher;
+
+        beforeEach(() => {
+            matcher = matchRoutes({ routes: ["api/foo"] });
+        });
+
+        it("matcher should be a function", () => {
+            expect(matcher).toBeInstanceOf(Function);
+        });
+
+        it("should recognize an exact route", () => {
+            const testPacket = { method: "POST", url: "api/foo" };
+            expect(matcher(testPacket)).toBe(true);
+        });
+
+        it("should recognize a subroute", () => {
+            const testPacket = { method: "POST", url: "api/foo/123" };
+            expect(matcher(testPacket)).toBe(true);
+        });
+
+        it("should recognize a subroute with some query params", () => {
+            const testPacket = { method: "POST", url: "api/foo/123?foo=bar" };
+            expect(matcher(testPacket)).toBe(true);
+        });
+
+        it("should not match unwatched urls", () => {
+            const testPacket = { method: "POST", url: "api/notinthelist" };
+            expect(matcher(testPacket)).toBe(false);
+        });
+
+        it("should not match unwatched methods", () => {
+            const testPacket = { method: "GET", url: "api/notinthelist" };
+            expect(matcher(testPacket)).toBe(false);
+        });
+    });
+
+    describe("matchRoutes: general regex", () => {
+        it("should match foo or bar", () => {
+            const matcher = matchRoutes({
+                routes: [/api\/(foo|bar)/gi],
+            });
+
+            const fooTest = { method: "POST", url: "api/foo/asdfasdf" };
+            expect(matcher(fooTest)).toBe(true);
+            const barTest = { method: "POST", url: "api/bar/asdfasdf" };
+            expect(matcher(barTest)).toBe(true);
+            const blechTest = { method: "POST", url: "api/blech/asdfasdf" };
+            expect(matcher(blechTest)).toBe(false);
+        });
+    });
+
+    describe("matchRoutes: excludes", () => {
+        const matcher = matchRoutes({
+            routes: ["api/histories"],
+            exclude: ["api/histories/ABC"],
+        });
+
+        // should exclude urls containing the restricted ID
+        it("should exclude history routes with id=ABC", () => {
+            const testRoute = { method: "POST", url: "api/histories/ABC" };
+            expect(matcher(testRoute)).toBe(false);
+            const testRouteWithParams = { method: "POST", url: "api/histories/ABC?hoohah=123" };
+            expect(matcher(testRouteWithParams)).toBe(false);
+        });
+
+        // should pass similar routes with different ids
+        it("should match other history routes", () => {
+            const testRoute = { method: "POST", url: "api/histories/DEF" };
+            expect(matcher(testRoute)).toBe(true);
+            const testRouteWithParams = { method: "POST", url: "api/histories/DEF?foo=sadfasd" };
+            expect(matcher(testRouteWithParams)).toBe(true);
+        });
+    });
+});

config/plugins/tours/core.galaxy_ui.yaml

@@ -1,4 +1,3 @@
-id: galaxy_ui
 name: Galaxy UI
 description: A gentle introduction to the Galaxy User Interface
 title_default: "Welcome to Galaxy"

config/plugins/tours/core.scratchbook.yaml

@@ -1,6 +1,6 @@
 name: Scratchbook - Introduction
-title_default: "Scratchbook Introduction"
 description: "An introduction on how to display multiple datasets and visualizations next to each other."
+title_default: "Scratchbook Introduction"
 tags:
   - "core"
   - "UI"

config/plugins/visualizations/nora/config/nora.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE visualization SYSTEM "../../visualization.dtd">
+<visualization name="Nora" embeddable="true">
+    <description>NORA's viewer. A medical image viewer and annotation tool.</description>
+    <data_sources>
+        <data_source>
+            <model_class>HistoryDatasetAssociation</model_class>
+            <test test_attr="ext" result_type="datatype">nrrd</test>
+            <test test_attr="ext" result_type="datatype">stl</test>
+            <test test_attr="ext" result_type="datatype">nii1</test>
+            <test test_attr="ext" result_type="datatype">nii1.gz</test>
+            <test test_attr="ext" result_type="datatype">gii</test>
+            <test test_attr="ext" result_type="datatype">gii.gz</test>
+            <test test_attr="ext" result_type="datatype">png</test>
+            <test test_attr="ext" result_type="datatype">jpg</test>
+            <to_param param_attr="id">dataset_id</to_param>
+        </data_source>
+    </data_sources>
+    <params>
+        <param type="dataset" var_name_in_template="hda" required="true">dataset_id</param>
+    </params>
+    <entry_point entry_point_type="chart" src="dist/script.js"/>
+    <specs>
+        <tag>svg</tag>
+        <use_panels>optional</use_panels>
+    </specs>
+</visualization>

config/plugins/visualizations/nora/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "nora",
+  "license": "MIT",
+  "description": "NORA Medical Image Viewer",
+  "version": "1.1.0",
+  "keywords": [
+    "galaxy",
+    "visualization"
+  ],
+  "main": "index.js",
+  "dependencies": {
+    "@galaxyproject/nora": "^1.1.0",
+    "cpy-cli": "^3.1.1"
+  },
+  "scripts": {
+    "build": "cpy --cwd='node_modules/@galaxyproject/nora/static/dist' --parents './' '../../../../../static/dist'"
+  }
+}

config/plugins/webhooks/demo/tour_generator/__init__.py

@@ -44,7 +44,7 @@ class TourGenerator:
         }
 
         # Conditional datasets
-        for name, value in self._test.inputs.items():
+        for name in self._test.inputs.keys():
             if '|' in name:
                 input_name = name.split('|')[1]
                 if input_name in self._data_inputs.keys():
@@ -119,8 +119,8 @@ class TourGenerator:
 
         steps = [{
             'title': tour_name,
-            'content': 'This short tour will guide you through the <b>' +
-                       self._tool.name + '</b> tool.',
+            'content': 'This short tour will guide you through the <b>'
+                       + self._tool.name + '</b> tool.',
             'orphan': True
         }]
 

contrib/nagios/check_galaxy.py

@@ -138,9 +138,9 @@ class Browser:
 
     def runtool(self):
         path = '/api/tools'
-        data = {'tool_id' : self.tool,
-                'history_id' : self.history_id,
-                'inputs' : {'echo' : self.handler}}
+        data = {'tool_id': self.tool,
+                'history_id': self.history_id,
+                'inputs': {'echo': self.handler}}
         res = self.req(path, data=data)
         dprint(json.loads(res.read()))
 
@@ -158,7 +158,7 @@ class Browser:
         return self._history_id
 
     def create_history(self):
-        res = self.req('/api/histories', data={'name' : handler})
+        res = self.req('/api/histories', data={'name': handler})
         self._history_id = json.loads(res.read())['id']
 
     @property

doc/Makefile

@@ -2,7 +2,7 @@
 #
 
 # You can set these variables from the command line.
-SPHINXOPTS    =
+SPHINXOPTS    ?=
 SPHINXBUILD   = sphinx-build
 PAPER         =
 BUILDDIR      = build
@@ -15,7 +15,7 @@ ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) sou
 # the i18n builder cannot share the environment and doctrees with the others
 I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
 
-GENERATED_RST = source/api/api.rst source/api/ts_api.rst source/dev/schema.md source/admin/config_logging_default_yaml.rst
+GENERATED_RST = source/dev/schema.md source/admin/config_logging_default_yaml.rst
 
 .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext updaterst
 
@@ -48,18 +48,6 @@ source/dev/run_tests_help.txt:
 source/dev/schema.md: parse_gx_xsd.py schema_template.md ../lib/galaxy/tool_util/xsd/galaxy.xsd ## Build Github-flavored Markdown from Galaxy Tool XSD (expects lxml in environment)
 	python parse_gx_xsd.py schema_template.md ../lib/galaxy/tool_util/xsd/galaxy.xsd > $@
 
-source/api/api.rst: source/lib/galaxy.webapps.galaxy.api.rst
-	printf "Galaxy API\n==========\n" > $@
-	tail -n +11 source/lib/galaxy.webapps.galaxy.api.rst >> $@
-	sed -i.bak -e 's/^galaxy\\\.webapps\\\.galaxy\\\.api\\\.//' $@
-	rm -f $@.bak
-
-source/api/ts_api.rst: source/lib/tool_shed.webapp.api.rst
-	printf "Tool Shed API\n=============\n" > $@
-	tail -n +11 source/lib/tool_shed.webapp.api.rst >> $@
-	sed -i.bak -e 's/^tool_shed\\\.webapp\\\.api\\\.//' $@
-	rm -f $@.bak
-
 source/admin/config_logging_default_yaml.rst: ../lib/galaxy/config/__init__.py
 	printf '.. code-block:: yaml\n\n' > $@
 	printf '    galaxy:\n' >> $@

doc/source/admin/cluster.md

@@ -2,7 +2,7 @@
 
 Galaxy is designed to run jobs on your local system by default, but it can be configured to run jobs on a cluster.  The front-end Galaxy application runs on a single server as usual, but tools are run on cluster nodes instead.
 
-A [general reference for the job configuration file](jobs.md) is also available.
+A [general reference for the job configuration file](./jobs.md) is also available.
 
 ## Distributed Resources Managers
 
@@ -70,7 +70,7 @@ You may also find that attribute caching in your filesystem causes problems with
 
 ## Runner Configuration
 
-**This documentation covers configuration of the various runner plugins, not how to distribute jobs to the various plugins.** Consult the [job configuration file documentation](jobs.md) for full details on the correct syntax, and for instructions on how to configure tools to actually use the runners explained below.
+**This documentation covers configuration of the various runner plugins, not how to distribute jobs to the various plugins.** Consult the [job configuration file documentation](./jobs.md) for full details on the correct syntax, and for instructions on how to configure tools to actually use the runners explained below.
 
 ### Local
 

doc/source/admin/galaxy_options.rst

@@ -677,8 +677,8 @@
 :Description:
     involucro is a tool used to build Docker or Singularity containers
     for tools from Conda dependencies referenced in tools as
-    `requirement`s. The following path is the location of involucro on
-    the Galaxy host. This is ignored if the relevant container
+    `requirement` s. The following path is the location of involucro
+    on the Galaxy host. This is ignored if the relevant container
     resolver isn't enabled, and will install on demand unless
     involucro_auto_init is set to false.
     The value of this option will be resolved with respect to
@@ -1049,6 +1049,21 @@
 :Type: str
 
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``enable_tool_document_cache``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Set this to false to disable the tool document cache. This cache
+    stores expanded xml strings. Disabling the tool cache results in
+    slower startup times. The tool cache is backed by sqlite database,
+    which cannot be stored on certain network disks. The cache
+    location is configurable using the ``tool_cache_data_dir``
+    setting, but can be disabled completely here.
+:Default: ``true``
+:Type: bool
+
+
 ~~~~~~~~~~~~~~~~~~~~~~~
 ``tool_cache_data_dir``
 ~~~~~~~~~~~~~~~~~~~~~~~
@@ -3709,7 +3724,7 @@
     you can separate Galaxy into multiple processes.  There are more
     than one way to do this, and they are explained in detail in the
     documentation:
-      https://docs.galaxyproject.org/en/master/admin/scaling.html
+    https://docs.galaxyproject.org/en/master/admin/scaling.html
     By default, Galaxy manages and executes jobs from within a single
     process and notifies itself of new jobs via in-memory queues.
     Jobs are run locally on the system on which Galaxy is started.
@@ -4085,7 +4100,7 @@
 :Description:
     Prologue Markdown/HTML to apply to markdown exports to PDF.
     Allowing branded headers.
-:Default: ````
+:Default: ``""``
 :Type: str
 
 
@@ -4096,7 +4111,7 @@
 :Description:
     Prologue Markdown/HTML to apply to markdown exports to PDF.
     Allowing branded footers.
-:Default: ````
+:Default: ``""``
 :Type: str
 
 
@@ -4107,7 +4122,7 @@
 :Description:
     Alternative to markdown_export_prologue that applies just to page
     exports.
-:Default: ````
+:Default: ``""``
 :Type: str
 
 
@@ -4118,7 +4133,7 @@
 :Description:
     Alternative to markdown_export_prologue that applies just to
     invocation report exports.
-:Default: ````
+:Default: ``""``
 :Type: str
 
 
@@ -4129,7 +4144,7 @@
 :Description:
     Alternative to markdown_export_epilogue that applies just to page
     exports.
-:Default: ````
+:Default: ``""``
 :Type: str
 
 
@@ -4140,7 +4155,7 @@
 :Description:
     Alternative to markdown_export_epilogue that applies just to
     invocation report exports.
-:Default: ````
+:Default: ``""``
 :Type: str
 
 

doc/source/admin/index.rst

@@ -18,6 +18,7 @@ This documentation is in the midst of being ported and unified based on resource
    jobs
    authentication
    tool_panel
+   mq
    dependency_resolvers
    conda_faq
    reports

doc/source/admin/mq.md

@@ -1,3 +1,4 @@
+
 # Message queue
 
 Galaxy uses a message queue internally for communicating between processes. 

doc/source/admin/nginx.md

@@ -399,7 +399,7 @@ these paths need to exposed in the same way.
 
 You may find it useful to require authentication for access to certain paths on your server.  For example, Galaxy can
 run a separate reports app which gives useful information about your Galaxy instance. See the [Reports Configuration
-documentation](reports) and [Peter Briggs' blog post on the
+documentation](./reports) and [Peter Briggs' blog post on the
 subject](http://galacticengineer.blogspot.com/2015/06/exposing-galaxy-reports-via-nginx-in.html) for more.
 
 After successfully following the blog post, Galaxy reports should be available at e.g. `https://galaxy.example.org/reports`.

doc/source/admin/proxy_package_layout.rst

@@ -1,3 +1,5 @@
+:orphan:
+
 Proxy Package Layouts
 ========================================
 

doc/source/admin/scaling.md

@@ -438,8 +438,8 @@ separated,  to the `job-handlers` farm. For example, 3 handlers are defined like
 
 By default, a job will be handled by whatever mule currently has the lock on the mule message queue. After receiving a
 message, it will release the lock, giving other mules a chance to handle future jobs.  Jobs can be explicitly mapped to
-specific mules as described in the [Job configuration documentation](jobs.md) by using the handler IDs
-`main.job-handlers.N`, where `N` is the mule's position in the farm, starting at 1 and incrementing for each mule in the
+specific mules as described in the [Galaxy Job Configuration](./jobs.md) documentation
+by using the handler IDs `main.job-handlers.N`, where `N` is the mule's position in the farm, starting at 1 and incrementing for each mule in the
 farm (this is not necessarily the mule ID, but it will be if you only define one farm and you add mules to that farm in
 sequential order).  Each worker that you wish to explicitly map jobs to should be defined in the `<handlers>` section
 of `job_conf.xml`. *Do not* define a default handler.

doc/source/admin/special_topics/index.rst

@@ -10,6 +10,7 @@ Special Topics
    interactive_environments
    mulled_containers
    grt
+   gtn
    job_metrics
    webhooks
    performance_tracking

doc/source/api/api.rst

@@ -0,0 +1,5 @@
+Galaxy API
+==========
+
+For a detailed description of the of Galaxy API controller please see
+the API module :py:mod:`galaxy.webapps.galaxy.api`.

doc/source/api/guidelines.rst

@@ -1,70 +0,0 @@
-API Design Guidelines
-=====================
-
-The following section outlines guidelines related to extending and/or modifying
-the Galaxy API. The Galaxy API has grown in an ad-hoc fashion over time by
-many contributors and so clients SHOULD NOT expect the API will conform to
-these guidelines - but developers contributing to the Galaxy API SHOULD follow
-these guidelines.
-
-    - API functionality should include docstring documentation for consumption
-      at docs.galaxyproject.org.
-    - Developers should familiarize themselves with the HTTP status code definitions
-      http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html. The API responses
-      should properly set the status code according to the result - in particular
-      2XX responses should be used for successful requests, 4XX for various
-      kinds of client errors, and 5XX for the errors on the server side.
-    - If there is an error processing some part of the request (one item in a list
-      for instance), the status code should be set to reflect the error and the
-      partial result may or may not be returned depending on the controller -
-      this behavior should be documented.
-    - API methods should throw a finite number of exceptions 
-      (defined in :mod:`galaxy.exceptions`) and these should subclass 
-      `MessageException` and not paste/wsgi HTTP exceptions. When possible, 
-      the framework itself should be responsible for catching these exceptions, 
-      setting the status code, and building an error response.
-    - Error responses should not consist of plain text strings - they should be
-      dictionaries describing the error and containing the following::
-
-          {
-            "status_code": 400,
-            "err_code": 400007,
-            "err_msg": "Request contained invalid parameter, action could not be completed.",
-            "type": "error",
-            "extra_error_info": "Extra information."
-          }
-
-      Various error conditions (once a format has been chosen and framework to
-      enforce it in place) should be spelled out in this document.
-    - Backward compatibility is important and should be maintained when possible.
-      If changing behavior in a non-backward compatible way please ensure one
-      of the following holds - there is a strong reason to believe no consumers
-      depend on a behavior, the behavior is effectively broken, or the API
-      method being modified has not been part of a tagged dist release.
-
-The following bullet points represent good practices more than guidelines, please
-consider them when modifying the API.
-
-    - Functionality should not be copied and pasted between controllers -
-      consider refactoring functionality into associated classes or short of
-      that into Mixins (http://en.wikipedia.org/wiki/Composition_over_inheritance)
-      or into Managers (:mod:`galaxy.managers`).
-    - API additions are more permanent changes to Galaxy than many other potential
-      changes and so a second opinion on API changes should be sought.
-    - New API functionality should include functional tests. These functional
-      tests should be implemented in Python and placed in
-      `lib/galaxy_test/api`.
-    - Changes to reflect modifications to the API should be pushed upstream to
-      the BioBlend project if possible.
-
-Longer term goals/notes.
-
-    - It would be advantageous to have a clearer separation of anonymous and
-      admin handling functionality.
-    - If at some point in the future, functionality needs to be added that
-      breaks backward compatibility in a significant way to a component used by
-      the community - a "dev" variant of the API will be established and
-      the community should be alerted and given a timeframe for when the old
-      behavior will be replaced with the new behavior.
-    - Consistent standards for range-based requests, batch requests, filtered
-      requests, etc... should be established and documented here.

doc/source/api/quickstart.rst

@@ -1,186 +1,6 @@
 Quickstart
 ==========
 
-Log in as your user, navigate to the API Keys page in the User menu, and
-generate a new API key. Make a note of the API key, and then pull up a
-terminal.  Now we'll use the display.py script in your ``galaxy/scripts/api``
-directory for a short example, which assumes your local galaxy server is
-running on the default port 8080::
-
-        % ./display.py my_key http://localhost:8080/api/histories
-        Collection Members
-        ------------------
-        #1: /api/histories/8c49be448cfe29bc
-          name: Unnamed history
-          id: 8c49be448cfe29bc
-        #2: /api/histories/33b43b4e7093c91f
-          name: output test
-          id: 33b43b4e7093c91f
-
-The result is a Collection of the histories of the user specified by the API
-key (you).  To look at the details of a particular history, say #1 above, do
-the following::
-
-        % ./display.py my_key http://localhost:8080/api/histories/8c49be448cfe29bc
-        Member Information
-        ------------------
-        state_details: {'ok': 1, 'failed_metadata': 0, 'upload': 0, 'discarded': 0, 'running': 0, 'setting_metadata': 0, 'error': 0, 'new': 0, 'queued': 0, 'empty': 0}
-        state: ok
-        contents_url: /api/histories/8c49be448cfe29bc/contents
-        id: 8c49be448cfe29bc
-        name: Unnamed history
-
-This gives detailed information about the specific member in question, in this
-case the History.  To view history contents, do the following::
-
-
-        % ./display.py my_key http://localhost:8080/api/histories/8c49be448cfe29bc/contents
-        Collection Members
-        ------------------
-        #1: /api/histories/8c49be448cfe29bc/contents/6f91353f3eb0fa4a
-          name: Pasted Entry
-          type: file
-          id: 6f91353f3eb0fa4a
-
-What we have here is another Collection of items containing all of the datasets
-in this particular history.  Finally, to view details of a particular dataset
-in this collection, execute the following::
-
-        % ./display.py my_key http://localhost:8080/api/histories/8c49be448cfe29bc/contents/6f91353f3eb0fa4a
-        Member Information
-        ------------------
-        misc_blurb: 1 line
-        name: Pasted Entry
-        data_type: txt
-        deleted: False
-        file_name: /Users/yoplait/work/galaxy-stock/database/files/000/dataset_82.dat
-        state: ok
-        download_url: /datasets/6f91353f3eb0fa4a/display?to_ext=txt
-        visible: True
-        genome_build: ?
-        model_class: HistoryDatasetAssociation
-        file_size: 17
-        metadata_data_lines: 1
-        id: 6f91353f3eb0fa4a
-        misc_info: uploaded txt file
-        metadata_dbkey: ?
-
-And now you've successfully used the API to request and select a history,
-browse the contents of that history, and then look at detailed information
-about a particular dataset.
-
-For a more comprehensive Data Library example, set the following options in your
-galaxy.yml as well, and restart galaxy again::
-
-        admin_users: you@example.org
-        library_import_dir: /path/to/some/directory
-
-In the directory you specified for 'library_import_dir', create some
-subdirectories, and put (or symlink) files to import into Galaxy into those
-subdirectories.
-
-In Galaxy, create an account that matches the address you put in 'admin_users',
-then browse to that user's preferences and generate a new API Key.  Copy the
-key to your clipboard and then use these scripts::
-
-        % ./display.py my_key http://localhost:8080/api/libraries
-        Collection Members
-        ------------------
-
-        0 elements in collection
-
-        % ./library_create_library.py my_key http://localhost:8080/api/libraries api_test 'API Test Library'
-        Response
-        --------
-        /api/libraries/f3f73e481f432006
-          name: api_test
-          id: f3f73e481f432006
-
-        % ./display.py my_key http://localhost:8080/api/libraries
-        Collection Members
-        ------------------
-        /api/libraries/f3f73e481f432006
-          name: api_test
-          id: f3f73e481f432006
-
-        % ./display.py my_key http://localhost:8080/api/libraries/f3f73e481f432006
-        Member Information
-        ------------------
-        synopsis: None
-        contents_url: /api/libraries/f3f73e481f432006/contents
-        description: API Test Library
-        name: api_test
-
-        % ./display.py my_key http://localhost:8080/api/libraries/f3f73e481f432006/contents
-        Collection Members
-        ------------------
-        /api/libraries/f3f73e481f432006/contents/28202595c0d2591f61ddda595d2c3670
-          name: /
-          type: folder
-          id: 28202595c0d2591f61ddda595d2c3670
-
-        % ./library_create_folder.py my_key http://localhost:8080/api/libraries/f3f73e481f432006/contents 28202595c0d2591f61ddda595d2c3670 api_test_folder1 'API Test Folder 1'
-        Response
-        --------
-        /api/libraries/f3f73e481f432006/contents/28202595c0d2591fa4f9089d2303fd89
-          name: api_test_folder1
-          id: 28202595c0d2591fa4f9089d2303fd89
-
-        % ./library_upload_from_import_dir.py my_key http://localhost:8080/api/libraries/f3f73e481f432006/contents 28202595c0d2591fa4f9089d2303fd89 bed bed hg19
-        Response
-        --------
-        /api/libraries/f3f73e481f432006/contents/e9ef7fdb2db87d7b
-          name: 2.bed
-          id: e9ef7fdb2db87d7b
-        /api/libraries/f3f73e481f432006/contents/3b7f6a31f80a5018
-          name: 3.bed
-          id: 3b7f6a31f80a5018
-
-        % ./display.py my_key http://localhost:8080/api/libraries/f3f73e481f432006/contents 
-        Collection Members
-        ------------------
-        /api/libraries/f3f73e481f432006/contents/28202595c0d2591f61ddda595d2c3670
-          name: /
-          type: folder
-          id: 28202595c0d2591f61ddda595d2c3670
-        /api/libraries/f3f73e481f432006/contents/28202595c0d2591fa4f9089d2303fd89
-          name: /api_test_folder1
-          type: folder
-          id: 28202595c0d2591fa4f9089d2303fd89
-        /api/libraries/f3f73e481f432006/contents/e9ef7fdb2db87d7b
-          name: /api_test_folder1/2.bed
-          type: file
-          id: e9ef7fdb2db87d7b
-        /api/libraries/f3f73e481f432006/contents/3b7f6a31f80a5018
-          name: /api_test_folder1/3.bed
-          type: file
-          id: 3b7f6a31f80a5018
-
-        % ./display.py my_key http://localhost:8080/api/libraries/f3f73e481f432006/contents/e9ef7fdb2db87d7b
-        Member Information
-        ------------------
-        misc_blurb: 68 regions
-        metadata_endCol: 3
-        data_type: bed
-        metadata_columns: 6
-        metadata_nameCol: 4
-        uploaded_by: nate@...
-        metadata_strandCol: 6
-        name: 2.bed
-        genome_build: hg19
-        metadata_comment_lines: None
-        metadata_startCol: 2
-        metadata_chromCol: 1
-        file_size: 4272
-        metadata_data_lines: 68
-        message:
-        metadata_dbkey: hg19
-        misc_info: uploaded bed file
-        date_uploaded: 2010-06-22T17:01:51.266119
-        metadata_column_types: str, int, int, str, int, str
-
-Other parameters are valid when uploading, they are the same parameters as are
-used in the web form, like 'link_data_only' and etc.
-
-The request and response format should be considered alpha and are subject to change.
-
+The Galaxy Training Network material has a tutorial designed
+to teach using the Galaxy API at
+https://training.galaxyproject.org/training-material/topics/dev/tutorials/bioblend-api/slides.html.

doc/source/api/ts_api.rst

@@ -0,0 +1,5 @@
+ToolShed API
+=============
+
+For a detailed description of the of Tool Shed API controller please see
+the API module :py:mod:`tool_shed.webapp.api`.

doc/source/api_doc.rst

@@ -28,7 +28,4 @@ Overview of available API resources is available at https://galaxyproject.org/de
 
     Quickstart <api/quickstart>
 
-    Development Guidelines <api/guidelines>
-
     Documentation <api/api>
-

doc/source/conf.py

@@ -18,6 +18,10 @@ import sphinx_rtd_theme
 # Library to make .md to slideshow
 from recommonmark.transform import AutoStructify
 
+# Set GALAXY_DOCS_SKIP_VIEW_CODE=1 to skip embedding highlighted source
+# code into docs.
+SKIP_VIEW_CODE = os.environ.get("GALAXY_DOCS_SKIP_VIEW_CODE", False) == "1"
+
 # Set GALAXY_DOCS_SKIP_SOURCE=1 to skip building source information and
 # just build primary documentation. (Quicker to debug issues in most frequently updated
 # docs).
@@ -43,7 +47,9 @@ sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pa
 extensions = ['recommonmark', 'sphinx.ext.intersphinx', 'sphinx_markdown_tables']
 if not SKIP_SOURCE:
     # TODO: Add https://pypi.org/project/sphinx-autodoc-typehints
-    extensions += ['sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode', 'sphinx.ext.autodoc']
+    extensions += ['sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.autodoc']
+    if not SKIP_VIEW_CODE:
+        extensions.append('sphinx.ext.viewcode')
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -134,7 +140,10 @@ pygments_style = 'sphinx'
 #modindex_common_prefix = []
 
 # Intersphinx mapping to Python documentation
-intersphinx_mapping = {'python': ('https://docs.python.org/3', None)}
+intersphinx_mapping = {
+    'python': ('https://docs.python.org/3', None),
+    'requests': ("https://requests.readthedocs.io/en/master/", None),
+}
 
 # -- Options for HTML output ---------------------------------------------------
 

doc/source/dev/api_guidelines.rst

@@ -0,0 +1,172 @@
+API Design Guidelines
+=====================
+
+The following section outlines guidelines related to extending and/or modifying
+the Galaxy API. The Galaxy API has been developed over time in an ad-hoc fashion and by
+many contributors, so clients SHOULD NOT expect the API will conform to
+these guidelines - but developers contributing to the Galaxy API SHOULD follow
+these guidelines.
+
+Before Getting Started
+~~~~~~~~~~~~~~~~~~~~~~
+
+Developers should familiarize themselves with `HTTP method definitions
+<https://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html#sec9>`__. While Galaxy
+follows these imperfectly - a serious effort should be made to use HTTP
+methods (GET, POST, PUT, DELETE, etc.) correctly and consistently. More
+information about these methods with a focus on RESTful APIs can be found
+at https://restfulapi.net/http-methods/.
+
+Developers should also familiarize themselves with the `HTTP status code definitions
+<http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html>`__. The API responses
+should properly set the status code according to the result - in particular
+2XX responses should be used for successful requests, 4XX for various
+kinds of client errors, and 5XX for the errors on the server side.
+
+Finally, developers should familiarize themselves with `RESTful API design
+<https://docs.microsoft.com/en-us/azure/architecture/best-practices/api-design>`__.
+Again, Galaxy follows generic RESTful guidelines imperfectly, but a serious
+effort should be made to design the API as RESTfully and consistently as possible.
+
+Documenting the API
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+An API endpoint implementation method should include high-level docstring documentation
+for consumption at docs.galaxyproject.org.
+
+API functionality should consume and produce Pydantic_ `models
+<https://pydantic-docs.helpmanual.io/usage/models/>`__ with typed and annotated
+`fields <https://pydantic-docs.helpmanual.io/usage/types/>`__. This information
+will be used to generate detailed OpenAPI_ documentation for APIs in
+the near future. Typed models also improve the quality and thoroughness
+of the static checking that can be done for the Galaxy backend.
+
+To further document your API and allow other developers to see how it functions,
+new API functionality should include functional tests. These functional
+tests should be implemented in Python and added to either the API test suite
+or the integration test suite. Checkout the :doc:`Writing Tests for Galaxy <writing_tests>`
+documentation for more information on developing tests for Galaxy.
+
+.. note::
+
+    The Galaxy documentation can be generated by running ``make docs-develop``
+    from the Galaxy root. Galaxy can be type checked by running ``tox -e mypy``
+    from the root of Galaxy after tox_ has been configured.
+
+Error Handling
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+API methods should throw a finite number of exceptions 
+(defined in :mod:`galaxy.exceptions`) and these should subclass 
+:class:`galaxy.exceptions.MessageException` and not paste/WSGI/FastAPI HTTP exceptions.
+When possible, the framework itself should be responsible for catching these exceptions,
+setting the status code, and building an error response. Throwing these exceptions
+will ensure the API responses are consistent across frameworks and across the API.
+These exceptions allow throwers to attach arbitrary extra information.
+
+Error responses should not consist of plain text strings - they should be
+dictionaries describing the error and containing the following:
+
+.. code-block: json
+
+    {
+      "status_code": 400,
+      "err_code": 400007,
+      "err_msg": "Request contained invalid parameter, action could not be completed.",
+      "type": "error",
+      "extra_error_info": "Extra information."
+    }
+
+The error code (``err_code``) is defined by the exception class
+and pulled from ``lig/galaxy/exceptions/error_codes.json``. This JSON
+document should be used to add new error conditions or improve
+default error messages.
+
+If there is an error processing some part of the request (one item in a list
+for instance), the status code should be set to reflect the error and the
+partial result may or may not be returned depending on the controller -
+this behavior should be documented.
+
+Backward Compatibility
+~~~~~~~~~~~~~~~~~~~~~~
+
+Backward compatibility is important and should be maintained when possible.
+If changing behavior in a non-backward compatible way please ensure one
+of the following holds - there is a strong reason to believe no consumers
+depend on the old behavior, the behavior is effectively broken, or the API
+method being modified has not been part of a tagged dist release. If these
+conditions do not hold, please help the Galaxy release manager ensure
+information about the breaking API is highlighted in the Galaxy release
+notes.
+
+Whenever functionality needs to be added, removed, or modified that
+breaks backward compatibility in a significant way to a component used by
+the community - a "dev" variant of the API should be established and
+the community should be alerted and given a time frame for when the old
+behavior will be replaced with the new behavior.
+
+Design
+~~~~~~
+
+Functionality should not be copied and pasted between controllers -
+consider refactoring functionality into associated classes or short of
+that into mixins_ or into managers (:mod:`galaxy.managers`).
+
+To whatever degree is possible, the Galaxy API controller classes
+(found in :mod:`galaxy.webapps.galaxy.api`) should be as lean as
+possible. Most of the logic should be pushed into components that
+are decoupled from the actual web application and the web framework.
+The role of the controller should be simply to adapt the web request
+(tied to the web framework) to simple objects (plain Python objects,
+Pydantic_ models, or SQLAlchemy models), send these objects to a
+Python component lower in the stack, and finally to translate the
+response to a web response. This design will ensure that Galaxy
+application logic is usable in as many contexts as possible and
+decouple the core "business logic" of Galaxy from the web framework
+hosting it.
+
+Going forward, we anticipate the most typical way this abstract
+advice translates to concrete code flow is that API controllers
+in :mod:`galaxy.webapps.galaxy.api` receive or produce Pydantic_
+models and pass them to manager components defined in
+:mod:`galaxy.managers`. The manager is then responsible for
+interacting with the whole backend or specific components to
+fullfil the desired request.
+
+Galaxy work contexts provide a way for API requests (and other
+components) to describe what they require from the framework
+in terms of the request (examples include the Galaxy app, a user
+to act on behalf of, or a user with a defined "current history").
+These objects describing the web request are passed around rather
+haphazardly as an argument named ``trans``. To design API methods
+to deal with ``trans`` in the most statically checkable and
+rigorous way - please review the docs for :mod:`galaxy.managers.context`.
+
+Examples
+~~~~~~~~
+
+A very clean, isolated example of a well documented controller
+and well annotated models is the licenses endpoint -
+:mod:`galaxy.webapps.galaxy.api.licenses`. A more complicated
+endpoint that demonstrates working with a manager and the
+Galaxy model backend is the roles endpoint - :mod:`galaxy.webapps.galaxy.api.roles`.
+
+BioBlend
+~~~~~~~~
+
+Changes to reflect modifications to the API should be pushed upstream to
+the BioBlend_ project if possible.
+
+Longer Term Goals
+~~~~~~~~~~~~~~~~~~
+
+    - It would be advantageous to have a clearer separation of anonymous and
+      admin handling functionality.
+    - Consistent standards for range-based requests, batch requests, filtered
+      requests, etc... should be established and documented here.
+
+.. _Pydantic: https://pydantic-docs.helpmanual.io/ 
+.. _BioBlend: https://github.com/galaxyproject/bioblend
+.. _mixins: http://en.wikipedia.org/wiki/Composition_over_inheritance
+.. _tox: https://tox.readthedocs.io/en/latest/
+.. _OpenAPI: https://www.openapis.org

doc/source/dev/index.rst

@@ -12,6 +12,7 @@ A multi-hour long video playlist covering these slides can be found at
   :maxdepth: 1
 
   schema
+  api_guidelines
   interactive_environments
   build_a_job_runner
   finding_and_improving_slow_code

doc/source/lib/galaxy.schema.rst

@@ -0,0 +1,26 @@
+galaxy.schema package
+=====================
+
+.. automodule:: galaxy.schema
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Submodules
+----------
+
+galaxy.schema.fields module
+---------------------------
+
+.. automodule:: galaxy.schema.fields
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+galaxy.schema.schema module
+---------------------------
+
+.. automodule:: galaxy.schema.schema
+   :members:
+   :undoc-members:
+   :show-inheritance:

doc/source/lib/galaxy.tool_util.client.rst

@@ -0,0 +1,18 @@
+galaxy.tool\_util.client package
+================================
+
+.. automodule:: galaxy.tool_util.client
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Submodules
+----------
+
+galaxy.tool\_util.client.staging module
+---------------------------------------
+
+.. automodule:: galaxy.tool_util.client.staging
+   :members:
+   :undoc-members:
+   :show-inheritance:

doc/source/lib/galaxy.tool_util.deps.rst

@@ -51,14 +51,6 @@ galaxy.tool\_util.deps.conda\_compat module
    :undoc-members:
    :show-inheritance:
 
-galaxy.tool\_util.deps.conda\_init module
------------------------------------------
-
-.. automodule:: galaxy.tool_util.deps.conda_init
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
 galaxy.tool\_util.deps.conda\_util module
 -----------------------------------------
 

doc/source/lib/galaxy.web.legacy_framework.rst

@@ -0,0 +1,18 @@
+galaxy.web.legacy\_framework package
+====================================
+
+.. automodule:: galaxy.web.legacy_framework
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Submodules
+----------
+
+galaxy.web.legacy\_framework.grids module
+-----------------------------------------
+
+.. automodule:: galaxy.web.legacy_framework.grids
+   :members:
+   :undoc-members:
+   :show-inheritance:

doc/source/lib/galaxy.webapps.reports.framework.rst

@@ -1,18 +0,0 @@
-galaxy.webapps.reports.framework package
-========================================
-
-.. automodule:: galaxy.webapps.reports.framework
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-Submodules
-----------
-
-galaxy.webapps.reports.framework.grids module
----------------------------------------------
-
-.. automodule:: galaxy.webapps.reports.framework.grids
-   :members:
-   :undoc-members:
-   :show-inheritance:

doc/source/lib/galaxy.workflow.refactor.rst

@@ -0,0 +1,26 @@
+galaxy.workflow.refactor package
+================================
+
+.. automodule:: galaxy.workflow.refactor
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Submodules
+----------
+
+galaxy.workflow.refactor.execute module
+---------------------------------------
+
+.. automodule:: galaxy.workflow.refactor.execute
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+galaxy.workflow.refactor.schema module
+--------------------------------------
+
+.. automodule:: galaxy.workflow.refactor.schema
+   :members:
+   :undoc-members:
+   :show-inheritance:

doc/source/releases/20.09-config.rst

@@ -1,3 +1,5 @@
+:orphan:
+
 Configuration Changes
 =====================
 

doc/source/releases/21.01_announce.rst

@@ -1,3 +1,4 @@
+:orphan:
 
 ===========================================================
 January 2021 Galaxy Release (v 21.01)

doc/source/releases/index.rst

@@ -4,12 +4,12 @@ Releases
 .. toctree::
    :maxdepth: 1
 
-   20.09_announce
-   20.05_announce
-   20.01_announce
-   19.09_announce
-   19.05_announce
-   19.01_announce
+   20.09_announce_user
+   20.05_announce_user
+   20.01_announce_user
+   19.09_announce_user
+   19.05_announce_user
+   19.01_announce_user
    18.09_announce
    18.05_announce
    18.01_announce

hooks/build

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+DOCKER_BUILDKIT=1 docker build -f $DOCKERFILE_PATH -t $IMAGE_NAME --build-arg GIT_USERNAME=$GIT_USERNAME --build-arg GIT_PASSWORD=$GIT_PASSWORD .

lib/galaxy/actions/library.py

@@ -85,7 +85,7 @@ class LibraryActions:
         populate_state(trans, tool.inputs, kwd, state.inputs)
         tool_params = state.inputs
         dataset_upload_inputs = []
-        for input_name, input in tool.inputs.items():
+        for input in tool.inputs.values():
             if input.type == "upload_dataset":
                 dataset_upload_inputs.append(input)
         # Library-specific params
@@ -203,7 +203,7 @@ class LibraryActions:
         if os.path.isfile(path):
             name = os.path.basename(path)
             files_and_folders.append((path, name, None))
-        for basedir, dirs, files in os.walk(line):
+        for basedir, _dirs, files in os.walk(line):
             for file in files:
                 file_path = os.path.abspath(os.path.join(basedir, file))
                 if preserve_dirs:
@@ -214,7 +214,7 @@ class LibraryActions:
         return files_and_folders
 
     def _paths_list(self, params):
-        return [(l.strip(), os.path.abspath(l.strip())) for l in params.get('filesystem_paths', '').splitlines() if l.strip()]
+        return [(line.strip(), os.path.abspath(line.strip())) for line in params.get('filesystem_paths', '').splitlines() if line.strip()]
 
     def _check_path_paste_params(self, params):
         if params.get('filesystem_paths', '') == '':

lib/galaxy/app.py

@@ -2,6 +2,7 @@ import logging
 import signal
 import sys
 import time
+from typing import Any
 
 import galaxy.model
 import galaxy.model.security
@@ -12,6 +13,7 @@ from galaxy.config_watchers import ConfigWatchers
 from galaxy.containers import build_container_interfaces
 from galaxy.files import ConfiguredFileSources
 from galaxy.job_metrics import JobMetrics
+from galaxy.managers.api_keys import ApiKeyManager
 from galaxy.managers.collections import DatasetCollectionManager
 from galaxy.managers.folders import FolderManager
 from galaxy.managers.hdas import HDAManager
@@ -43,7 +45,7 @@ from galaxy.tools.cache import (
 from galaxy.tools.data_manager.manager import DataManagers
 from galaxy.tools.error_reports import ErrorReports
 from galaxy.tools.special_tools import load_lib_tools
-from galaxy.tours import ToursRegistry
+from galaxy.tours import build_tours_registry
 from galaxy.util import (
     ExecutionTimer,
     heartbeat,
@@ -57,15 +59,16 @@ from galaxy.web.proxy import ProxyManager
 from galaxy.web_stack import application_stack_instance
 from galaxy.webhooks import WebhooksRegistry
 from galaxy.workflow.trs_proxy import TrsProxy
+from .structured_app import StructuredApp
 
 log = logging.getLogger(__name__)
 app = None
 
 
-class UniverseApplication(config.ConfiguresGalaxyMixin):
+class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin):
     """Encapsulates the state of a Universe application"""
 
-    def __init__(self, **kwargs):
+    def __init__(self, **kwargs) -> None:
         if not log.handlers:
             # Paste didn't handle it, so we need a temporary basic log
             # configured.  The handler added here gets dumped and replaced with
@@ -73,12 +76,11 @@ class UniverseApplication(config.ConfiguresGalaxyMixin):
             logging.basicConfig(level=logging.DEBUG)
         log.debug("python path is: %s", ", ".join(sys.path))
         self.name = 'galaxy'
-        # is_webapp will be set to true when building WSGI app
         self.is_webapp = False
-        self.startup_timer = ExecutionTimer()
+        startup_timer = ExecutionTimer()
         self.new_installation = False
         # Read config file and check for errors
-        self.config = config.Configuration(**kwargs)
+        self.config: Any = config.Configuration(**kwargs)
         self.config.check()
         config.configure_logging(self.config)
         self.execution_timer_factory = ExecutionTimerFactory(self.config)
@@ -116,6 +118,7 @@ class UniverseApplication(config.ConfiguresGalaxyMixin):
         self.library_manager = LibraryManager()
         self.role_manager = RoleManager(self)
         self.dynamic_tool_manager = DynamicToolManager(self)
+        self.api_keys_manager = ApiKeyManager(app)
 
         # ConfiguredFileSources
         self.file_sources = ConfiguredFileSources.from_app_config(self.config)
@@ -173,7 +176,7 @@ class UniverseApplication(config.ConfiguresGalaxyMixin):
             directories_setting=self.config.visualization_plugins_directory,
             template_cache_dir=self.config.template_cache_path)
         # Tours registry
-        self.tour_registry = ToursRegistry(self.config.tour_config_dir)
+        self.tour_registry = build_tours_registry(self.config.tour_config_dir)
         # Webhooks registry
         self.webhooks_registry = WebhooksRegistry(self.config.webhooks_dir)
         # Load security policy.
@@ -270,7 +273,7 @@ class UniverseApplication(config.ConfiguresGalaxyMixin):
         self.url_for = url_for
 
         self.server_starttime = int(time.time())  # used for cachebusting
-        log.info("Galaxy app startup finished %s" % self.startup_timer)
+        log.info("Galaxy app startup finished %s" % startup_timer)
 
     def shutdown(self):
         log.debug('Shutting down')

lib/galaxy/auth/providers/ldap_ad.py

@@ -144,20 +144,22 @@ class LDAP(AuthProvider):
 
         if 'search-fields' in options:
             try:
-                l = ldap.initialize(_get_subs(options, 'server', params))
-                l.protocol_version = 3
+                conn = ldap.initialize(_get_subs(options, 'server', params))
+                conn.protocol_version = 3
 
                 if 'search-user' in options:
-                    l.simple_bind_s(_get_subs(options, 'search-user', params),
-                                    _get_subs(options, 'search-password', params))
+                    conn.simple_bind_s(
+                        _get_subs(options, 'search-user', params),
+                        _get_subs(options, 'search-password', params))
                 else:
-                    l.simple_bind_s()
+                    conn.simple_bind_s()
 
                 # setup search
                 attributes = {_.strip().format(**params) for _ in options['search-fields'].split(',')}
                 if 'search-memberof-filter' in options:
                     attributes.add('memberOf')
-                suser = l.search_ext_s(_get_subs(options, 'search-base', params),
+                suser = conn.search_ext_s(
+                    _get_subs(options, 'search-base', params),
                     ldap.SCOPE_SUBTREE,
                     _get_subs(options, 'search-filter', params), attributes,
                     timeout=60, sizelimit=1)
@@ -230,17 +232,17 @@ class LDAP(AuthProvider):
         Do the actual authentication by binding as the user to check their credentials
         """
         try:
-            l = ldap.initialize(_get_subs(options, 'server', params))
-            l.protocol_version = 3
+            conn = ldap.initialize(_get_subs(options, 'server', params))
+            conn.protocol_version = 3
             bind_user = _get_subs(options, 'bind-user', params)
             bind_password = _get_subs(options, 'bind-password', params)
         except Exception:
             log.exception('LDAP authenticate: initialize exception')
             return False
         try:
-            l.simple_bind_s(bind_user, bind_password)
+            conn.simple_bind_s(bind_user, bind_password)
             try:
-                whoami = l.whoami_s()
+                whoami = conn.whoami_s()
             except ldap.PROTOCOL_ERROR:
                 # The "Who am I?" extended operation is not supported by this LDAP server
                 pass

lib/galaxy/authnz/__init__.py

@@ -26,12 +26,14 @@ class IdentityProvider:
 
         :type config: lxml.etree.ElementTree._Element
         :param config: Is the configuration element of the provider
-            from the configuration file (e.g., oidc_config.xml).
-            This element contains the all the provider-specific
-            configuration elements.
+                       from the configuration file (e.g., oidc_config.xml).
+                       This element contains the all the provider-specific
+                       configuration elements.
 
         :type backend_config: lxml.etree.ElementTree._Element
-        :param backend_config: Is the configuration element of the backend of
+        :param backend_config:
+
+            Is the configuration element of the backend of
             the provider from the configuration file (e.g.,
             oidc_backends_config.xml). This element contains all the
             backend-specific configuration elements.
@@ -47,24 +49,23 @@ class IdentityProvider:
         :type trans: GalaxyWebTransaction
         :param trans: Galaxy web transaction.
 
-        :return: a redirect URI to the provider's authentication
-            endpoint.
+        :returns: a redirect URI to the provider's authentication endpoint
         """
         raise NotImplementedError()
 
-    def callback(self, state_token, authz_code, trans, login_redirect_url):
-        """
-        Handles authentication call-backs from identity providers.
-        This process maps `state-token` to a user
-        :type state_token: string
+    def callback(self, state_token: str, authz_code: str, trans, login_redirect_url):
+        """Handles authentication call-backs from identity providers.
+
+        This process maps `state-token` to a user.
+
         :param state_token: is an anti-forgery token which identifies
-            a Galaxy user to whom the given authorization code belongs to.
-        :type authz_code: string
+                            a Galaxy user to whom the given authorization code belongs to.
         :param authz_code: a very short-lived, single-use token to
-            request a refresh token.
+                           request a refresh token.
         :type trans: GalaxyWebTransaction
         :param trans: Galaxy web transaction.
-        :return tuple: a tuple of redirect_url and user.
+        :rtype: tuple
+        :returns: a tuple of redirect_url and user.
         """
         raise NotImplementedError()
 
@@ -79,7 +80,7 @@ class IdentityProvider:
         :type trans: GalaxyWebTransaction
         :param trans: Galaxy web transaction.
 
-        :type trans: string
-        :param trans: Optional URL to redirect to after logging out of IDP.
+        :type post_logout_redirect_url: string
+        :param post_logout_redirect_url: Optional URL to redirect to after logging out of IDP.
         """
         raise NotImplementedError()

lib/galaxy/authnz/custos_authnz.py

@@ -102,9 +102,9 @@ class CustosAuthnz(IdentityProvider):
                     # TODO: Future work will expand on this and provide an
                     # interface for when there are multiple auth providers
                     # allowing explicit authenticated association.
-                    if (trans.app.config.enable_oidc and
-                            len(trans.app.config.oidc) == 1 and
-                            len(trans.app.auth_manager.authenticators) == 0):
+                    if (trans.app.config.enable_oidc
+                            and len(trans.app.config.oidc) == 1
+                            and len(trans.app.auth_manager.authenticators) == 0):
                         user = existing_user
                     else:
                         message = "There already exists a user with email %s.  To associate this external login, you must first be logged in as that existing account." % email

lib/galaxy/authnz/psa_authnz.py

@@ -282,9 +282,10 @@ def contains_required_data(response=None, is_new=False, **kwargs):
     and returns void if otherwise.
 
     :type  response: dict
-    :param response:    a dictionary containing decoded response from
-                        OIDC backend that contain the following keys
-                        among others:
+    :param response:  a dictionary containing decoded response from
+                      OIDC backend that contain the following keys
+                      among others:
+
                         -   id_token;       see: http://openid.net/specs/openid-connect-core-1_0.html#IDToken
                         -   access_token;   see: https://tools.ietf.org/html/rfc6749#section-1.4
                         -   refresh_token;  see: https://tools.ietf.org/html/rfc6749#section-1.5
@@ -297,6 +298,7 @@ def contains_required_data(response=None, is_new=False, **kwargs):
     :param is_new: has the user been authenticated?
 
     :param kwargs:      may contain the following keys among others:
+
                         -   uid:        user ID
                         -   user:       Galaxy user; if user is already authenticated
                         -   backend:    the backend that is used for user authentication.

lib/galaxy/config/config_manage.py

@@ -426,6 +426,8 @@ def _write_option_rst(args, rst, key, heading_level, option_value):
         default = "true"
     elif default is False:
         default = "false"
+    elif default == "":
+        default = '""'
     rst.write(":Default: ``%s``\n" % default)
     if type:
         rst.write(":Type: %s\n" % type)
@@ -686,7 +688,7 @@ def _build_sample_yaml(args, app_desc):
     description = getattr(schema, "description", None)
     if description:
         description = description.lstrip()
-        as_comment = "\n".join("# %s" % l for l in description.split("\n")) + "\n"
+        as_comment = "\n".join(f"# {line}" for line in description.split("\n")) + "\n"
         f.write(as_comment)
     _write_sample_section(args, f, 'uwsgi', Schema(UWSGI_OPTIONS), as_comment=False, uwsgi_hack=True)
     _write_sample_section(args, f, app_desc.app_name, schema)
@@ -700,7 +702,7 @@ def _write_to_file(args, f, path):
     else:
         contents = f
     if args.dry_run:
-        contents_indented = "\n".join(" |%s" % l for l in contents.splitlines())
+        contents_indented = "\n".join(f" |{line}" for line in contents.splitlines())
         print(f"Overwriting {path} with the following contents:\n{contents_indented}")
         print("... skipping because --dry-run is enabled.")
     else:
@@ -755,7 +757,7 @@ def _write_option(args, f, key, option_value, as_comment=False, uwsgi_hack=False
     else:
         key_val_str = yaml.dump({key: value}, width=float("inf")).lstrip("{").rstrip("\n}")
     lines = f"{comment}{as_comment_str}{key_val_str}"
-    lines_idented = "\n".join("  %s" % l for l in lines.split("\n"))
+    lines_idented = "\n".join(f"  {line}" for line in lines.split("\n"))
     f.write("%s\n\n" % lines_idented)
 
 

lib/galaxy/config/sample/container_resolvers_conf.xml.sample

@@ -52,7 +52,7 @@
        or more package requirements listed as part of the tool's definition.
        Set auto_install to False if Galaxy should build container images
        through the admin interface or API, but not automatically when
-       a tool is run.
+       a tool is run. WARNING: requires docker to be installed.
 
   -->
 

lib/galaxy/config/sample/galaxy.yml.sample

@@ -436,7 +436,7 @@ galaxy:
 
   # involucro is a tool used to build Docker or Singularity containers
   # for tools from Conda dependencies referenced in tools as
-  # `requirement`s. The following path is the location of involucro on
+  # `requirement` s. The following path is the location of involucro on
   # the Galaxy host. This is ignored if the relevant container resolver
   # isn't enabled, and will install on demand unless involucro_auto_init
   # is set to false.
@@ -617,6 +617,14 @@ galaxy:
   # generated commands run in sh.
   #default_job_shell: /bin/bash
 
+  # Set this to false to disable the tool document cache. This cache
+  # stores expanded xml strings. Disabling the tool cache results in
+  # slower startup times. The tool cache is backed by sqlite database,
+  # which cannot be stored on certain network disks. The cache location
+  # is configurable using the ``tool_cache_data_dir`` setting, but can
+  # be disabled completely here.
+  #enable_tool_document_cache: true
+
   # Tool related caching. Fully expanded tools and metadata will be
   # stored at this path. Per tool_conf cache locations can be configured
   # in (``shed_``)tool_conf.xml files using the tool_cache_data_dir
@@ -1830,7 +1838,7 @@ galaxy:
   # can separate Galaxy into multiple processes.  There are more than
   # one way to do this, and they are explained in detail in the
   # documentation:
-  #   https://docs.galaxyproject.org/en/master/admin/scaling.html
+  # https://docs.galaxyproject.org/en/master/admin/scaling.html
   # By default, Galaxy manages and executes jobs from within a single
   # process and notifies itself of new jobs via in-memory queues.  Jobs
   # are run locally on the system on which Galaxy is started.  Advanced

lib/galaxy/config/script.py

@@ -154,7 +154,8 @@ def _handle_galaxy_yml(args, config_dir, data_dir):
 
     galaxy_config_template = []
     with open(GALAXY_CONFIG_TEMPLATE_FILE) as fh:
-        for line in [l.rstrip('\n') for l in fh.readlines()]:
+        for line in fh:
+            line = line.rstrip('\n')
             for k, v in GALAXY_CONFIG_SUBSTITUTIONS.items():
                 if line == k:
                     line = v

lib/galaxy/containers/__init__.py

@@ -14,8 +14,7 @@ from abc import (
     abstractmethod,
     abstractproperty
 )
-from collections import namedtuple
-from typing import Any, Dict, Optional, Type
+from typing import Any, Dict, NamedTuple, Optional, Type
 
 import yaml
 
@@ -29,18 +28,12 @@ DEFAULT_CONF = {'_default_': {'type': DEFAULT_CONTAINER_TYPE}}
 log = logging.getLogger(__name__)
 
 
-class ContainerPort(namedtuple('ContainerPort', ('port', 'protocol', 'hostaddr', 'hostport'))):
-    """Named tuple representing ports published by a container, with attributes:
-
-    :ivar       port:       Port number (inside the container)
-    :vartype    port:       int
-    :ivar       protocol:   Port protocol, either ``tcp`` or ``udp``
-    :vartype    protocol:   str
-    :ivar       hostaddr:   Address or hostname where the published port can be accessed
-    :vartype    hostaddr:   str
-    :ivar       hostport:   Published port number on which the container can be accessed
-    :vartype    hostport:   int
-    """
+class ContainerPort(NamedTuple):
+    """Named tuple representing ports published by a container, with attributes"""
+    port: int  # Port number (inside the container)
+    protocol: str  # Port protocol, either ``tcp`` or ``udp``
+    hostaddr: str  # Address or hostname where the published port can be accessed
+    hostport: int  # Published port number on which the container can be accessed
 
 
 class ContainerVolume(metaclass=ABCMeta):
@@ -80,12 +73,15 @@ class ContainerVolume(metaclass=ABCMeta):
 class Container(metaclass=ABCMeta):
 
     def __init__(self, interface, id, name=None, **kwargs):
-        """:param   interface:  Container interface for the given container type
+        """
+
+        :param      interface:  Container interface for the given container type
         :type       interface:  :class:`ContainerInterface` subclass instance
         :param      id:         Container identifier
         :type       id:         str
         :param      name:       Container name
         :type       name:       str
+
         """
         self._interface = interface
         self._id = id
@@ -118,7 +114,7 @@ class Container(metaclass=ABCMeta):
         """Attribute for accessing details of ports published by the container.
 
         :returns:   Port details
-        :rtype:     list of :class:`ContainerPort`s
+        :rtype:     list of :class:`ContainerPort` s
         """
 
     @abstractproperty
@@ -246,25 +242,25 @@ class ContainerInterface(metaclass=ABCMeta):
     def _stringify_kwopt_list_of_kvpairs(self, flag, val):
         """
         """
-        l = []
+        kwopt_list = []
         if isinstance(val, list):
             # ['foo=bar', 'baz=quux']
-            l = val
+            kwopt_list = val
         else:
             # {'foo': 'bar', 'baz': 'quux'}
             for k, v in dict(val).items():
-                l.append(f'{k}={v}')
-        return self._stringify_kwopt_list(flag, l)
+                kwopt_list.append(f'{k}={v}')
+        return self._stringify_kwopt_list(flag, kwopt_list)
 
     def _stringify_kwopt_list_of_kovtrips(self, flag, val):
         """
         """
         if isinstance(val, str):
             return self._stringify_kwopt_string(flag, val)
-        l = []
+        kwopt_list = []
         for k, o, v in val:
-            l.append(f'{k}{o}{v}')
-        return self._stringify_kwopt_list(flag, l)
+            kwopt_list.append(f'{k}{o}{v}')
+        return self._stringify_kwopt_list(flag, kwopt_list)
 
     def _run_command(self, command, verbose=False):
         if verbose:
@@ -381,7 +377,7 @@ def _get_interface_modules():
     modules = import_submodules(sys.modules[__name__])
     for module in modules:
         module_names = [getattr(module, _) for _ in dir(module)]
-        classes = [_ for _ in module_names if inspect.isclass(_) and
-            not _ == ContainerInterface and issubclass(_, ContainerInterface)]
+        classes = [_ for _ in module_names if inspect.isclass(_)
+            and not _ == ContainerInterface and issubclass(_, ContainerInterface)]
         interfaces.extend(classes)
     return {x.container_type: x for x in interfaces}

lib/galaxy/containers/docker.py

@@ -152,25 +152,25 @@ class DockerCLIInterface(DockerInterface):
         """The docker API will take a volumes argument in many formats, try to
         deal with that for the command line
         """
-        l = []
+        kwopt_list = []
         if isinstance(val, list):
             # ['/host/vol']
-            l = val
+            kwopt_list = val
         else:
             for hostvol, guestopts in val.items():
                 if isinstance(guestopts, str):
                     # {'/host/vol': '/container/vol'}
-                    l.append(f'{hostvol}:{guestopts}')
+                    kwopt_list.append(f'{hostvol}:{guestopts}')
                 else:
                     # {'/host/vol': {'bind': '/container/vol'}}
                     # {'/host/vol': {'bind': '/container/vol', 'mode': 'rw'}}
                     mode = guestopts.get('mode', '')
-                    l.append('{vol}:{bind}{mode}'.format(
+                    kwopt_list.append('{vol}:{bind}{mode}'.format(
                         vol=hostvol,
                         bind=guestopts['bind'],
                         mode=':' + mode if mode else ''
                     ))
-        return self._stringify_kwopt_list(flag, l)
+        return self._stringify_kwopt_list(flag, kwopt_list)
 
     def _run_docker(self, subcommand, args=None, verbose=False):
         command = self._docker_command.format(subcommand=subcommand, args=args or '')
@@ -265,7 +265,6 @@ class DockerAPIClient:
         for tries in range(1, max_tries + 1):
             retry_time = DockerAPIClient._exception_retry_time
             reinit = False
-            exc = None
             # re-get the APIClient method every time as a different caller (such as the success test function) may have
             # already reinitialized the client, and we always want to use the current client
             f = DockerAPIClient._unwrapped_attr(fname)
@@ -275,42 +274,38 @@ class DockerAPIClient:
                 if tries > 1:
                     log.info('%s() succeeded on attempt %s', qualname, tries)
                 return r
-            except ConnectionError:
-                reinit = True
-            except docker.errors.APIError as exc:
-                if not DockerAPIClient._should_retry_request(exc.response.status_code):
-                    raise
-            except ReadTimeout:
-                reinit = True
-                retry_time = 0
-            finally:
-                # this is inside the finally context so we can do a bare raise when we give up (so the real stack for
-                # the exception is raised)
-                if exc is not None:
-                    log.warning("Caught exception on %s(): %s: %s",
-                                DockerAPIClient._qualname(f), exc.__class__.__name__, exc)
-                    if reinit:
-                        log.warning("Reinitializing Docker API client due to connection-oriented failure")
-                        DockerAPIClient._init_client()
-                        f = DockerAPIClient._unwrapped_attr(fname)
-                        qualname = DockerAPIClient._qualname(f)
-                    r = None
-                    if success_test is not None:
-                        log.info("Testing if %s() succeeded despite the exception", qualname)
-                        r = success_test()
-                    if r:
-                        log.warning("The request appears to have succeeded, will not retry. Response is: %s", str(r))
-                        return r
-                    elif tries >= max_tries:
-                        log.error("Maximum number of attempts (%s) exceeded", max_tries)
-                        if 'response' in exc and DockerAPIClient._nonfatal_error(exc.response.status_code):
-                            return None
-                        else:
-                            raise
+            except (ConnectionError, docker.errors.APIError, ReadTimeout) as exc:
+                if isinstance(exc, ConnectionError):
+                    reinit = True
+                elif isinstance(exc, docker.errors.APIError):
+                    if not DockerAPIClient._should_retry_request(exc.response.status_code):
+                        raise
+                else:  # ReadTimeout
+                    reinit = True
+                    retry_time = 0
+                log.warning("Caught exception on %s(): %s: %s",
+                            DockerAPIClient._qualname(f), exc.__class__.__name__, exc)
+                if reinit:
+                    log.warning("Reinitializing Docker API client due to connection-oriented failure")
+                    DockerAPIClient._init_client()
+                    f = DockerAPIClient._unwrapped_attr(fname)
+                    qualname = DockerAPIClient._qualname(f)
+                r = None
+                if success_test is not None:
+                    log.info("Testing if %s() succeeded despite the exception", qualname)
+                    r = success_test()
+                if r:
+                    log.warning("The request appears to have succeeded, will not retry. Response is: %s", str(r))
+                    return r
+                elif tries >= max_tries:
+                    log.error("Maximum number of attempts (%s) exceeded", max_tries)
+                    if 'response' in exc and DockerAPIClient._nonfatal_error(exc.response.status_code):
+                        return None
                     else:
-                        log.error("Retrying %s() in %s seconds (attempt: %s of %s)", qualname, retry_time, tries,
-                                  max_tries)
-                        sleep(retry_time)
+                        raise
+                else:
+                    log.error("Retrying %s() in %s seconds (attempt: %s of %s)", qualname, retry_time, tries, max_tries)
+                    sleep(retry_time)
 
     def __init__(self, *args, **kwargs):
         # Only initialize the host iterator once

lib/galaxy/datatypes/_schema.py

@@ -0,0 +1,129 @@
+from typing import (
+    Dict,
+    List,
+    Optional,
+)
+
+from pydantic import (
+    BaseModel,
+    Field,
+    HttpUrl
+)
+
+
+class CompositeFileInfo(BaseModel):
+    name: str = Field(
+        ...,  # Mark this field as required
+        title="Name",
+        description="The name of this composite file"
+    )
+    optional: bool = Field(
+        title="Optional",
+        description=""  # TODO add description
+    )
+    mimetype: Optional[str] = Field(
+        title="MIME type",
+        description="The MIME type of this file"
+    )
+    description: Optional[str] = Field(
+        title="Description",
+        description="Summary description of the purpouse of this file"
+    )
+    substitute_name_with_metadata: Optional[str] = Field(
+        title="Substitute name with metadata",
+        description=""  # TODO add description
+    )
+    is_binary: bool = Field(
+        title="Is binary",
+        description="Whether this file is a binary file"
+    )
+    to_posix_lines: bool = Field(
+        title="To posix lines",
+        description=""  # TODO add description
+    )
+    space_to_tab: bool = Field(
+        title="Spaces to tabulation",
+        description=""  # TODO add description
+    )
+
+
+class DatatypeDetails(BaseModel):
+    extension: str = Field(
+        ...,  # Mark this field as required
+        title="Extension",
+        description="The data type’s Dataset file extension",
+        example="bed"
+    )
+    description: Optional[str] = Field(
+        title="Description",
+        description="A summary description for this data type"
+    )
+    description_url: Optional[HttpUrl] = Field(
+        title="Description URL",
+        description="The URL to a detailed description for this datatype",
+        example="https://wiki.galaxyproject.org/Learn/Datatypes#Bed"
+    )
+    display_in_upload: bool = Field(
+        default=False,
+        title="Display in upload",
+        description="If True, the associated file extension will be displayed in the `File Format` select list in the `Upload File from your computer` tool in the `Get Data` tool section of the tool panel"
+    )
+    composite_files: Optional[List[CompositeFileInfo]] = Field(
+        default=None,
+        title="Composite files",
+        description="A collection of files composing this data type"
+    )
+
+
+class DatatypesMap(BaseModel):
+    ext_to_class_name: Dict[str, str] = Field(
+        ...,  # Mark this field as required
+        title="Extension Map",
+        description="Dictionary mapping datatype's extensions with implementation classes",
+    )
+    class_to_classes: Dict[str, Dict[str, bool]] = Field(
+        ...,  # Mark this field as required
+        title="Classes Map",
+        description="Dictionary mapping datatype's classes with their base classes",
+    )
+
+
+class DatatypesCombinedMap(BaseModel):
+    datatypes: List[str] = Field(
+        ...,  # Mark this field as required
+        title="Datatypes",
+        description="List of datatypes extensions",
+    )
+    datatypes_mapping: DatatypesMap = Field(
+        ...,  # Mark this field as required
+        title="Datatypes Mapping",
+        description="Dictionaries for mapping datatype's extensions/classes with their implementation classes",
+    )
+
+
+class DatatypeConverter(BaseModel):
+    source: str = Field(
+        ...,  # Mark this field as required
+        title="Source",
+        description="Source type for conversion",
+        example="bam",
+    )
+    target: str = Field(
+        ...,  # Mark this field as required
+        title="Target",
+        description="Target type for conversion",
+        example="bai",
+    )
+    tool_id: str = Field(
+        ...,  # Mark this field as required
+        title="Tool identifier",
+        description="The converter tool identifier",
+        example="CONVERTER_Bam_Bai_0",
+    )
+
+
+class DatatypeConverterList(BaseModel):
+    __root__: List[DatatypeConverter] = Field(
+        title='List of data type converters',
+        default=[]
+    )

lib/galaxy/datatypes/converters/fasta_to_len.py

@@ -49,5 +49,5 @@ def compute_fasta_length(fasta_file, out_file, keep_first_char, keep_first_word=
         out.write("%s\t%d\n" % (fasta_title[1:keep_first_char], seq_len))
 
 
-if __name__ == "__main__" :
+if __name__ == "__main__":
     compute_fasta_length(sys.argv[1], sys.argv[2], sys.argv[3], True)

lib/galaxy/datatypes/converters/fasta_to_tabular_converter.py

@@ -45,5 +45,5 @@ def __main__():
                 out.write("".join(sequence.split()) + '\n')
 
 
-if __name__ == "__main__" :
+if __name__ == "__main__":
     __main__()