Merge branch 'dev' into webpack5

.github/workflows/api.yaml

@@ -28,11 +28,20 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Get full Python version
+        id: full-python-version
+        shell: bash
+        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
       - name: Cache pip dir
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
           key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+      - name: Cache galaxy venv
+        uses: actions/cache@v2
+        with:
+          path: .venv
+          key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-api
       - name: Run tests
         run: ./run_tests.sh --skip_flakey_fails -api
         working-directory: 'galaxy root'

.github/workflows/api_paste.yaml

@@ -30,11 +30,20 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Get full Python version
+        id: full-python-version
+        shell: bash
+        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
       - name: Cache pip dir
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
           key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+      - name: Cache galaxy venv
+        uses: actions/cache@v2
+        with:
+          path: .venv
+          key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-api-paste
       - name: Run tests
         run: ./run_tests.sh --skip_flakey_fails -api
         working-directory: 'galaxy root'

.github/workflows/db_indexes.yaml

@@ -34,12 +34,21 @@ jobs:
     - uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Get full Python version
+      id: full-python-version
+      shell: bash
+      run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
     - name: Cache pip dir
       uses: actions/cache@v1
       id: pip-cache
       with:
         path: ~/.cache/pip
         key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+    - name: Cache tox env
+      uses: actions/cache@v2
+      with:
+        path: .tox
+        key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-check-indexes
     - name: Install tox
       run: pip install tox
     - name: Check indexes on postgresql

.github/workflows/first_startup.yaml

@@ -29,12 +29,21 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Get full Python version
+      id: full-python-version
+      shell: bash
+      run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
     - name: Cache pip dir
       uses: actions/cache@v1
       id: pip-cache
       with:
         path: ~/.cache/pip
         key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+    - name: Cache tox env
+      uses: actions/cache@v2
+      with:
+        path: .tox
+        key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-first-startup
     - uses: mvdbeek/gha-yarn-cache@master
       with:
         yarn-lock-file: 'galaxy root/client/yarn.lock'

.github/workflows/framework.yaml

@@ -28,11 +28,20 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Get full Python version
+        id: full-python-version
+        shell: bash
+        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
       - name: Cache pip dir
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
           key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+      - name: Cache galaxy venv
+        uses: actions/cache@v2
+        with:
+          path: .venv
+          key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-framework
       - name: Run tests
         run: ./run_tests.sh --framework
         working-directory: 'galaxy root'

.github/workflows/integration.yaml

@@ -53,12 +53,21 @@ jobs:
       - uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Get full Python version
+        id: full-python-version
+        shell: bash
+        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
       - name: Cache pip dir
         uses: actions/cache@v1
         id: pip-cache
         with:
           path: ~/.cache/pip
           key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+      - name: Cache galaxy venv
+        uses: actions/cache@v2
+        with:
+          path: .venv
+          key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-integration
       - name: Install ffmpeg
         run: sudo apt-get update && sudo apt-get -y install ffmpeg
         if: matrix.subset == 'upload_datatype'

.github/workflows/integration_selenium.yaml

@@ -33,12 +33,21 @@ jobs:
       - uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Get full Python version
+        id: full-python-version
+        shell: bash
+        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
       - name: Cache pip dir
         uses: actions/cache@v1
         id: pip-cache
         with:
           path: ~/.cache/pip
           key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+      - name: Cache galaxy venv
+        uses: actions/cache@v2
+        with:
+          path: .venv
+          key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-selenium
       - uses: mvdbeek/gha-yarn-cache@master
         with:
           yarn-lock-file: 'galaxy root/client/yarn.lock'

.github/workflows/lint.yaml

@@ -15,11 +15,20 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Get full Python version
+        id: full-python-version
+        shell: bash
+        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
       - name: Cache pip dir
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
           key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('lib/galaxy/dependencies/pinned-lint-requirements.txt') }}
+      - name: Cache tox env
+        uses: actions/cache@v2
+        with:
+          path: .tox
+          key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-lint
       - name: Install tox
         run: pip install tox
       - name: Run linting

.github/workflows/mulled.yaml

@@ -17,12 +17,21 @@ jobs:
       - uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Get full Python version
+        id: full-python-version
+        shell: bash
+        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
       - name: Cache pip dir
         uses: actions/cache@v1
         id: pip-cache
         with:
           path: ~/.cache/pip
           key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+      - name: Cache tox env
+        uses: actions/cache@v2
+        with:
+          path: .tox
+          key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-mulled
       - name: Install tox
         run: pip install tox
       - name: run tests

.github/workflows/osx_startup.yaml

@@ -14,6 +14,10 @@ jobs:
     - uses: actions/checkout@v2
       with:
         path: 'galaxy root'
+    - name: Get full Python version
+      id: full-python-version
+      shell: bash
+      run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
     - name: Cache pip dir
       uses: actions/cache@v1
       id: pip-cache
@@ -21,6 +25,11 @@ jobs:
         path: ~/Library/Caches/pip
         # scripts/common_startup.sh creates a conda env for Galaxy containing Python 3.6
         key: pip-cache-3.6-${{ hashFiles('galaxy root/requirements.txt') }}
+    - name: Cache tox env
+      uses: actions/cache@v2
+      with:
+        path: .tox
+        key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-osx
     - name: Install and activate miniconda  # use this job to test using Python from a conda environment
       uses: conda-incubator/setup-miniconda@v2
       with:

.github/workflows/performance.yaml

@@ -28,11 +28,20 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Get full Python version
+        id: full-python-version
+        shell: bash
+        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
       - name: Cache pip dir
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
           key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+      - name: Cache galaxy venv
+        uses: actions/cache@v2
+        with:
+          path: .venv
+          key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-performance
       - name: Run tests
         run: ./run_tests.sh --ci_test_metrics --structured_data_html --structured_data_report_file "test.json" --skip_flakey_fails -api lib/galaxy_test/performance
         working-directory: 'galaxy root'

.github/workflows/selenium.yaml

@@ -33,6 +33,10 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Get full Python version
+        id: full-python-version
+        shell: bash
+        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
       - name: Cache pip dir
         uses: actions/cache@v2
         with:
@@ -41,6 +45,11 @@ jobs:
       - uses: mvdbeek/gha-yarn-cache@master
         with:
           yarn-lock-file: 'galaxy root/client/yarn.lock'
+      - name: Cache galaxy venv
+        uses: actions/cache@v2
+        with:
+          path: .venv
+          key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-selenium
       - uses: nanasess/setup-chromedriver@master
       - name: Run tests
         run: ./run_tests.sh -selenium lib/galaxy_test/selenium -- --num-shards=3 --shard-id=${{ matrix.chunk }}

.github/workflows/selenium_beta.yaml

@@ -31,11 +31,20 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Get full Python version
+        id: full-python-version
+        shell: bash
+        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
       - name: Cache pip dir
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
           key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+      - name: Cache galaxy venv
+        uses: actions/cache@v2
+        with:
+          path: .venv
+          key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-selenium-beta
       - uses: nanasess/setup-chromedriver@master
       - name: Run tests
         run: ./run_tests.sh -selenium lib/galaxy_test/selenium -- --num-shards=3 --shard-id=${{ matrix.chunk }}

.github/workflows/toolshed.yaml

@@ -29,12 +29,21 @@ jobs:
       - uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Get full Python version
+        id: full-python-version
+        shell: bash
+        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
       - name: Cache pip dir
         uses: actions/cache@v1
         id: pip-cache
         with:
           path: ~/.cache/pip
           key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+      - name: Cache galaxy venv
+        uses: actions/cache@v2
+        with:
+          path: .venv
+          key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-toolshed
       - name: Run tests
         run: './run_tests.sh -toolshed'
         working-directory: 'galaxy root'

.github/workflows/unit.yaml

@@ -18,11 +18,20 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Get full Python version
+        id: full-python-version
+        shell: bash
+        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
       - name: Cache pip dir
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
           key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+      - name: Cache tox env
+        uses: actions/cache@v2
+        with:
+          path: .tox
+          key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-unit
       - name: Install ffmpeg
         run: sudo apt-get update && sudo apt-get -y install ffmpeg
       - name: Install tox

client/src/app/singleton.js

@@ -39,5 +39,5 @@ export function galaxyIsInitialized() {
 // Having a CORS issue in the toolshed iframe, store separate versions
 // of galaxy in each window for the short-term
 export function getStorage() {
-    return window.parent;
+    return window;
 }

client/src/components/Details/DatasetDetails.vue

@@ -6,6 +6,7 @@
                     v-if="!isDatasetLoading"
                     :jobid="dataset.creating_job"
                     v-slot="{ result: job, loading: isJobLoading }"
+                    :use-cache="false"
                 >
                     <div v-if="!isJobLoading">
                         <dataset-information class="detail" :hda_id="datasetId" />

client/src/components/Form/utilities.js

@@ -4,7 +4,7 @@ import _ from "underscore";
  * @param{dict}   inputs    - Nested dictionary of input elements
  * @param{dict}   callback  - Called with the mapped dictionary object and corresponding model node
  */
-export var visitInputs = (inputs, callback, prefix, context) => {
+export function visitInputs(inputs, callback, prefix, context) {
     context = Object.assign({}, context);
     _.each(inputs, (input) => {
         if (input && input.type && input.name) {
@@ -41,24 +41,32 @@ export var visitInputs = (inputs, callback, prefix, context) => {
                 callback(node, name, context);
         }
     }
-};
+}
 
 /** Match conditional values to selected cases
  * @param{dict}   input     - Definition of conditional input parameter
  * @param{dict}   value     - Current value
  */
-export var matchCase = (input, value) => {
+export function matchCase(input, value) {
     if (input.test_param.type == "boolean") {
-        if (value == "true") {
-            value = input.test_param.truevalue || "true";
+        if (["true", true].includes(value)) {
+            if (input.test_param.truevalue !== undefined) {
+                value = input.test_param.truevalue;
+            } else {
+                value = "true";
+            }
         } else {
-            value = input.test_param.falsevalue || "false";
+            if (input.test_param.falsevalue !== undefined) {
+                value = input.test_param.falsevalue;
+            } else {
+                value = "false";
+            }
         }
     }
-    for (var i in input.cases) {
+    for (let i = 0; i < input.cases.length; i++) {
         if (input.cases[i].value == value) {
             return i;
         }
     }
     return -1;
-};
+}

client/src/components/Form/utilities.test.js

@@ -0,0 +1,86 @@
+import { matchCase, visitInputs } from "./utilities";
+
+function visitInputsString(inputs) {
+    let results = "";
+    visitInputs(inputs, (input, identifier) => {
+        results += `${identifier}=${input.value};`;
+    });
+    return results;
+}
+
+describe("form component utilities", () => {
+    it("conditional case matching", () => {
+        const input = {
+            name: "a",
+            type: "conditional",
+            test_param: {
+                name: "b",
+                type: "boolean",
+                value: "true",
+                truevalue: undefined,
+                falsevalue: undefined,
+            },
+            cases: [
+                {
+                    value: "true",
+                    inputs: [
+                        {
+                            name: "c",
+                            type: "text",
+                            value: "cvalue",
+                        },
+                    ],
+                },
+                {
+                    value: "false",
+                    inputs: [
+                        {
+                            name: "d",
+                            type: "text",
+                            value: "dvalue",
+                        },
+                    ],
+                },
+            ],
+        };
+
+        // test simple case matching
+        expect(matchCase(input, "true")).toEqual(0);
+        expect(matchCase(input, true)).toEqual(0);
+        expect(matchCase(input, "false")).toEqual(1);
+        expect(matchCase(input, false)).toEqual(1);
+
+        // test truevalue
+        input.test_param.truevalue = "truevalue";
+        expect(matchCase(input, "true")).toEqual(-1);
+        input.cases[0].value = "truevalue";
+        expect(matchCase(input, "true")).toEqual(0);
+
+        // test falsevalue
+        input.test_param.falsevalue = "falsevalue";
+        expect(matchCase(input, "true")).toEqual(0);
+        expect(matchCase(input, "false")).toEqual(-1);
+        input.cases[1].value = "falsevalue";
+        expect(matchCase(input, "false")).toEqual(1);
+
+        // test (empty) truevalue
+        input.test_param.truevalue = undefined;
+        input.cases[0].value = "true";
+        expect(matchCase(input, "true")).toEqual(0);
+        input.test_param.truevalue = "";
+        expect(matchCase(input, "true")).toEqual(-1);
+        input.cases[0].value = "";
+        expect(matchCase(input, "true")).toEqual(0);
+
+        // test visit inputs
+        expect(visitInputsString([input])).toEqual("a|b=true;a|c=cvalue;");
+        input.test_param.value = "false";
+        expect(visitInputsString([input])).toEqual("a|b=false;a|d=dvalue;");
+
+        // switch test parameter to other type than boolean e.g. select
+        input.test_param.type = "select";
+        expect(matchCase(input, "")).toEqual(0);
+        expect(matchCase(input, "unavailable")).toEqual(-1);
+        expect(matchCase(input, "falsevalue")).toEqual(1);
+    });
+});

client/src/components/History/ContentItem/Dataset/DatasetMenu.vue

@@ -129,7 +129,7 @@
                     v-if="notIn(STATES.NOT_VIEWABLE)"
                     key="dataset-details"
                     title="View Dataset Details"
-                    @click.stop.prevent="backboneRoute(dataset.getUrl('show_params'))"
+                    @click.stop.prevent="showDetails"
                 >
                     <Icon icon="info-circle" class="mr-1" />
                     <span v-localize>View Dataset Details</span>
@@ -273,6 +273,17 @@ export default {
             }
         },
 
+        showDetails() {
+            const redirectParams = {
+                path: this.dataset.getUrl("show_params"),
+                title: "Dataset details",
+                tryIframe: false,
+            };
+            if (!this.iframeAdd(redirectParams)) {
+                this.backboneRoute(this.dataset.getUrl("show_params"));
+            }
+        },
+
         showToolHelp(job_id) {
             this.eventHub.$emit("toggleToolHelp", job_id);
         },

client/src/components/History/ContentItem/Dataset/Summary/Error.vue

@@ -3,7 +3,7 @@
         <div v-if="!props.dataset.purged && props.dataset.misc_info">
             <span>{{ props.dataset.misc_info }}</span>
         </div>
-        <span class="help-text" v-localize>An error occurred with this props.dataset</span>
+        <span class="help-text" v-localize>An error occurred with this dataset</span>
         <div v-if="props.dataset.misc_info" class="job-error-text">
             <span>{{ props.dataset.misc_info }}</span>
         </div>

client/src/components/History/caching/CacheWorker.worker.js

@@ -1,37 +0,0 @@
-/**
- * Instantiates worker with with externally defined api
- */
-
-// TODO: isn't @babel/polyfill bad now?
-import "@babel/polyfill";
-import { expose } from "threads/worker";
-import { asObservable } from "./asObservable";
-import { configure } from "./workerConfig";
-import * as api from "./CacheApi";
-
-const {
-    monitorContentQuery,
-    monitorDscQuery,
-    monitorHistoryContent,
-    monitorCollectionContent,
-    loadHistoryContents,
-    loadDscContent,
-    pollHistory,
-    ...promises
-} = api;
-
-expose({
-    configure,
-
-    // observable operators
-    monitorContentQuery: asObservable(monitorContentQuery),
-    monitorDscQuery: asObservable(monitorDscQuery),
-    monitorHistoryContent: asObservable(monitorHistoryContent),
-    monitorCollectionContent: asObservable(monitorCollectionContent),
-    loadHistoryContents: asObservable(loadHistoryContents),
-    loadDscContent: asObservable(loadDscContent),
-    pollHistory: asObservable(pollHistory),
-
-    // promise functions
-    ...promises,
-});

client/src/components/History/caching/loadDscContent.js

@@ -8,7 +8,7 @@ import { nth } from "utils/observable";
 import { requestWithUpdateTime } from "./operators/requestWithUpdateTime";
 import { bulkCacheDscContent } from "./db";
 import { SearchParams } from "../model/SearchParams";
-import { prependPath } from "./workerConfig";
+import { prependPath } from "utils/redirect";
 import { summarizeCacheOperation, dateStore } from "./loadHistoryContents";
 import { show } from "utils/observable";
 

client/src/components/History/caching/loadHistoryContents.js

@@ -3,7 +3,7 @@ import { map, pluck, share, filter } from "rxjs/operators";
 import { hydrate } from "utils/observable";
 import { areDefined } from "utils/validation";
 import { requestWithUpdateTime } from "./operators/requestWithUpdateTime";
-import { prependPath } from "./workerConfig";
+import { prependPath } from "utils/redirect";
 import { bulkCacheContent } from "./db";
 import { SearchParams } from "../model/SearchParams";
 import { createDateStore } from "../model/DateStore";

client/src/components/History/caching/workerClient.js

@@ -1,119 +0,0 @@
-import { defer, from, of, pipe } from "rxjs";
-import { filter, finalize, materialize, map, mergeMap, shareReplay, mergeAll } from "rxjs/operators";
-import { v4 as uuidv4 } from "uuid";
-import { spawn } from "threads";
-import CacheWorker from "./CacheWorker.worker.js";
-import config from "config";
-import { getRootFromIndexLink } from "onload/getRootFromIndexLink";
-
-/**
- * @constant Observable yields the worker thread instance
- */
-// prettier-ignore
-const threadInstance$ = defer(() => of(config).pipe(
-    mergeMap(buildThread)
-));
-
-const thread$ = threadInstance$.pipe(shareReplay(1));
-
-const buildThread = async (cfg) => {
-    const thread = await spawn(new CacheWorker());
-    if (!thread) {
-        throw new MissingWorkerError();
-    }
-
-    // Configure the worker This is sending in settings that are derived from
-    // galaxy's absurd global application instance or written directly to
-    // the document, which will not be available in the worker.
-    const root = getRootFromIndexLink();
-    const workerConfigs = { ...cfg, root };
-    await thread.configure(workerConfigs);
-
-    return thread;
-};
-
-// glorified pluck operator
-// prettier-ignore
-const method = (fnName) => pipe(
-    map((thread) => {
-        if (!(fnName in thread)) {
-            throw new MissingWorkerMethodError(fnName);
-        }
-        return thread[fnName];
-    })
-);
-
-/**
- * Give this a string of the function name on the worker thread instance,
- * returns an observable operator that transparently calls a matching oprator
- * from inside the worker .
- *
- * @param {string} fnName Name of an exposed property on the thread object
- */
-// prettier-ignore
-export const toOperator = (fnName) => {
-    const method$ = thread$.pipe(method(fnName));
-
-    // Result of the returned method call will be an "ObservablePromise", a
-    // custom object returned by thread library that the author probably thought
-    // was clever. We need to fix that by turning it back to a real observable
-
-    const cleanMethod$ = method$.pipe(
-        map((f) => (...args) => from(f(...args))) // I'm a real boy now!
-    );
-
-    const operator = (cfg = {}) => (src$) => {
-        // identifies subscription so we can match external observable with
-        // itnernal observable
-        const id = uuidv4();
-
-        return cleanMethod$.pipe(
-            mergeMap((method) => src$.pipe(
-                materialize(),
-                map((notification) => method({ id, cfg, fnName, ...notification })),
-                // first emission will be the observable created by threads
-                // that's the only one we want, rest should be nulls
-                filter(Boolean),
-                // subscribe to the observable threads made
-                mergeAll(),
-                // unsub when exterior observable completes
-                finalize(() => method({ id, kind: "C" }))
-            ))
-        );
-    };
-
-    return operator;
-};
-
-/**
- * Returns an async function from a property on the thread object. This is
- * actually what was already there, but we're using thread$ to manage the
- * lifetime of the worker instance, so we'll derive the function from the thread
- * observable.
- *
- * @param {string} workerMethod Name of method inside the worker
- * @return {Function} Function that returns a promise
- */
-export const toPromise = (fnName) => {
-    return async (...request) => {
-        const methodPromise = thread$.pipe(method(fnName)).toPromise();
-        const fn = await methodPromise;
-        return await fn(...request);
-    };
-};
-
-/**
- * Custom Errors
- */
-
-class MissingWorkerError extends Error {}
-
-class MissingWorkerMethodError extends Error {
-    constructor(missingMethod, ...args) {
-        const msg = `
-            Missing method on client cache worker: ${missingMethod}.
-            Please write a function named ${missingMethod} in caching/cacheWorker.js
-        `;
-        super(msg, ...args);
-    }
-}

client/src/components/History/caching/workerConfig.js

@@ -1,20 +0,0 @@
-/**
- * Configuration var for inside the worker, must be set when worker
- * is fired up because we can't reach the document and that's how galaxy
- * backbone code gets some of its config.
- */
-export const workerConfig = { root: "/" };
-
-export const configure = (options = {}) => {
-    Object.assign(workerConfig, options);
-};
-
-/**
- * Prepend against this config. Can't access document so we can't use
- * the standard one from utils
- */
-const slashCleanup = /(\/)+/g;
-export function prependPath(path) {
-    const root = workerConfig.root;
-    return `${root}/${path}`.replace(slashCleanup, "/");
-}

client/src/components/HistoryExport/ExportLink.vue

@@ -14,7 +14,7 @@
         <i
             title="Information about when the history export was generated is included in the job details. Additionally, if there are issues with export, the job details may help figure out the underlying problem or communicate issues to your Galaxy administrator."
         >
-            (<a class="show-job-link" href="#" @click="showDetails">view job details</a>)
+            (<b-link class="show-job-link" href="#" @click="showDetails">view job details</b-link>)
         </i>
         <b-modal v-model="details" modal-class="job-information-modal" scrollable ok-only hide-header>
             <job-information :job_id="historyExport.job_id" :include-times="true" />
@@ -55,7 +55,7 @@ export default {
             this.details = true;
         },
         copyUrl() {
-            copy(this.latestExportUrl, "Export URL copied to your clipboard");
+            copy(this.link, "Export URL copied to your clipboard");
         },
     },
 };

client/src/components/Markdown/Elements/HistoryDatasetCollection/CollectionDisplay.vue

@@ -131,5 +131,6 @@ export default {
 <style scoped>
 .content-height {
     max-height: 15rem;
+    overflow-y: auto;
 }
 </style>

client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue

@@ -177,5 +177,6 @@ export default {
 <style scoped>
 .content-height {
     max-height: 20rem;
+    overflow-y: auto;
 }
 </style>

client/src/components/Markdown/Elements/ToolStd.vue

@@ -33,5 +33,6 @@ export default {
 <style scoped>
 .content-height {
     max-height: 15rem;
+    overflow-y: auto;
 }
 </style>

client/src/components/Markdown/Elements/Workflow/WorkflowDisplay.vue

@@ -109,5 +109,6 @@ export default {
 <style scoped>
 .content-height {
     max-height: 15rem;
+    overflow-y: auto;
 }
 </style>

client/src/components/NewUserWelcome/NewUserWelcome.vue

@@ -1,5 +1,5 @@
 <template>
-    <div>
+    <div class="new-user-welcome">
         <ConfigProvider v-slot="config">
             <component
                 :is="viewElement"
@@ -85,93 +85,95 @@ export default {
 };
 </script>
 
-<style type="text/css">
-.card {
-    border: 0px;
-}
-.card-img {
-    height: 12rem;
-}
-.card-header,
-.card-footer {
-    border-bottom: 0px;
-    border-top: 0px;
-    background-color: #ffffff;
-}
-.carousel-fig {
-    padding-bottom: 10;
-}
-.carousel-caption {
-    position: relative;
-    left: 0;
-    top: 0;
-    color: black;
-    padding-bottom: 10;
-    padding-left: 1vw;
-    padding-right: 1vw;
-}
-.carousel-item {
-    float: none;
-    margin-right: auto;
-    margin-left: auto;
-}
-.carousel-indicators {
-    position: sticky;
-    bottom: 30px;
-}
-.button-housing {
-    position: sticky;
-    z-index: 21;
-    padding-right: 67.88px;
-}
-.carousel-button {
-    position: fixed;
-    z-index: 20;
-    bottom: 10px;
-}
-.carousel-control-next,
-.carousel-control-prev,
-.carousel-indicators {
-    filter: invert(100%);
-}
-.carousel-control-next,
-.carousel-control-prev {
-    border-left: 1px;
-    border-right: 1px;
-    z-index: 11;
-    align-items: center;
-    justify-content: center;
-    height: fit-content;
-    height: 85vh;
-    width: 2vw;
-}
-.carousel-control-next:hover,
-.carousel-control-prev:hover {
-    background-color: grey;
-}
-#logos img {
-    max-width: 100%;
-}
-.carousel-inner {
-    position: absolute;
-}
-.mini-img {
-    max-width: 100px;
-}
-.small-img {
-    max-width: 300px;
-}
-.med-img {
-    max-width: 500px;
-}
-.large-img {
-    max-width: 700px;
-}
-.slide-header {
-    text-align: center;
-    padding-bottom: 3;
-}
-.section-header {
-    filter: invert(16%) sepia(14%) saturate(1113%) hue-rotate(189deg) brightness(99%) contrast(91%);
+<style scoped type="scss">
+.new-user-welcome::v-deep {
+    .card {
+        border: 0px;
+    }
+    .card-img {
+        height: 12rem;
+    }
+    .card-header,
+    .card-footer {
+        border-bottom: 0px;
+        border-top: 0px;
+        background-color: #ffffff;
+    }
+    .carousel-fig {
+        padding-bottom: 10;
+    }
+    .carousel-caption {
+        position: relative;
+        left: 0;
+        top: 0;
+        color: black;
+        padding-bottom: 10;
+        padding-left: 1vw;
+        padding-right: 1vw;
+    }
+    .carousel-item {
+        float: none;
+        margin-right: auto;
+        margin-left: auto;
+    }
+    .carousel-indicators {
+        position: sticky;
+        bottom: 30px;
+    }
+    .button-housing {
+        position: sticky;
+        z-index: 21;
+        padding-right: 67.88px;
+    }
+    .carousel-button {
+        position: fixed;
+        z-index: 20;
+        bottom: 10px;
+    }
+    .carousel-control-next,
+    .carousel-control-prev,
+    .carousel-indicators {
+        filter: invert(100%);
+    }
+    .carousel-control-next,
+    .carousel-control-prev {
+        border-left: 1px;
+        border-right: 1px;
+        z-index: 11;
+        align-items: center;
+        justify-content: center;
+        height: fit-content;
+        height: 85vh;
+        width: 2vw;
+    }
+    .carousel-control-next:hover,
+    .carousel-control-prev:hover {
+        background-color: grey;
+    }
+    #logos img {
+        max-width: 100%;
+    }
+    .carousel-inner {
+        position: absolute;
+    }
+    .mini-img {
+        max-width: 100px;
+    }
+    .small-img {
+        max-width: 300px;
+    }
+    .med-img {
+        max-width: 500px;
+    }
+    .large-img {
+        max-width: 700px;
+    }
+    .slide-header {
+        text-align: center;
+        padding-bottom: 3;
+    }
+    .section-header {
+        filter: invert(16%) sepia(14%) saturate(1113%) hue-rotate(189deg) brightness(99%) contrast(91%);
+    }
 }
 </style>

client/src/components/Tool/ToolForm.vue

@@ -14,14 +14,16 @@
                         :tool-name="toolName"
                     />
                     <Webhook v-if="showSuccess" type="tool" :tool-id="jobDef.tool_id" />
-                    <b-alert v-if="showError" show variant="danger">
-                        <h4>{{ errorTitle | l }}</h4>
-                        <p>
-                            The server could not complete the request. Please contact the Galaxy Team if this error
-                            persists.
-                        </p>
-                        <pre>{{ errorContentPretty }}</pre>
-                    </b-alert>
+                    <b-modal v-model="showError" size="sm" :title="errorTitle | l" scrollable ok-only>
+                        <b-alert show variant="danger">
+                            The server could not complete this request. Please verify your parameter settings, retry
+                            submission and contact the Galaxy Team if this error persists. A transcript of the submitted
+                            data is shown below.
+                        </b-alert>
+                        <small class="text-muted">
+                            <pre>{{ errorContentPretty }}</pre>
+                        </small>
+                    </b-modal>
                     <ToolRecommendation v-if="showRecommendation" :tool-id="formConfig.id" />
                     <ToolCard
                         v-if="showForm"
@@ -163,12 +165,19 @@ export default {
         };
     },
     created() {
-        this.requestTool();
+        this.requestTool().then(() => {
+            const Galaxy = getGalaxyInstance();
+            if (Galaxy && Galaxy.currHistoryPanel) {
+                console.debug(`ToolForm::created - Started listening to history changes. [${this.id}]`);
+                Galaxy.currHistoryPanel.collection.on("change", this.onHistoryChange, this);
+            }
+        });
+    },
+    beforeDestroy() {
         const Galaxy = getGalaxyInstance();
         if (Galaxy && Galaxy.currHistoryPanel) {
-            Galaxy.currHistoryPanel.collection.on("change", () => {
-                this.onUpdate();
-            });
+            Galaxy.currHistoryPanel.collection.off("change", this.onHistoryChange, this);
+            console.debug(`ToolForm::beforeDestroy - Stopped listening to history changes. [${this.id}]`);
         }
     },
     computed: {
@@ -203,6 +212,10 @@ export default {
         reuseAllowed(user) {
             return allowCachedJobs(user.preferences);
         },
+        onHistoryChange() {
+            console.debug(`ToolForm::created - Loading history changes. [${this.id}]`);
+            this.onUpdate();
+        },
         onValidation(validationInternal) {
             this.validationInternal = validationInternal;
         },
@@ -225,7 +238,7 @@ export default {
         },
         requestTool(newVersion) {
             this.currentVersion = newVersion || this.currentVersion;
-            getToolFormData(this.id, this.currentVersion, this.job_id, this.history_id).then((data) => {
+            return getToolFormData(this.id, this.currentVersion, this.job_id, this.history_id).then((data) => {
                 this.formConfig = data;
                 this.remapAllowed = this.job_id && data.job_remap;
                 this.showLoading = false;
@@ -263,22 +276,24 @@ export default {
             console.debug("toolForm::onExecute()", jobDef);
             submitJob(jobDef).then(
                 (jobResponse) => {
+                    this.showExecuting = false;
                     if (Galaxy.currHistoryPanel) {
                         Galaxy.currHistoryPanel.refreshContents();
                     }
-                    this.showForm = false;
                     if (jobResponse.produces_entry_points) {
                         this.showEntryPoints = true;
                         this.entryPoints = jobResponse.jobs;
                     }
                     const nJobs = jobResponse && jobResponse.jobs ? jobResponse.jobs.length : 0;
                     if (nJobs > 0) {
+                        this.showForm = false;
                         this.showSuccess = true;
                         this.jobDef = jobDef;
                         this.jobResponse = jobResponse;
                     } else {
                         this.showError = true;
-                        this.errorTitle = "Invalid success response. No jobs found.";
+                        this.showForm = true;
+                        this.errorTitle = "Job submission rejected.";
                         this.errorContent = jobResponse;
                     }
                     if ([true, "true"].includes(config.enable_tool_recommendations)) {
@@ -299,8 +314,7 @@ export default {
                     }
                     if (genericError) {
                         this.showError = true;
-                        this.showForm = false;
-                        this.errorTitle = "Job submission failed";
+                        this.errorTitle = "Job submission failed.";
                         this.errorContent = this.jobDef;
                     }
                 }

client/src/components/Workflow/Editor/Forms/FormTool.vue

@@ -110,9 +110,6 @@ export default {
                         input.hiddenInWorkflow = true;
                         input.info = `Data input '${input.name}' (${Utils.textify(input.extensions)})`;
                         input.value = { __class__: "RuntimeValue" };
-                    } else if (input.type == "conditional") {
-                        input.connectable = false;
-                        input.test_param.collapsible_value = undefined;
                     } else if (!input.fixed) {
                         input.connectable = true;
                         input.collapsible_value = {
@@ -124,6 +121,12 @@ export default {
                     }
                 }
             });
+            Utils.deepeach(inputs, (input) => {
+                if (input.type === "conditional") {
+                    input.connectable = false;
+                    input.test_param.collapsible_value = undefined;
+                }
+            });
             return inputs;
         },
         errors() {

client/src/components/Workflow/Editor/Node.vue

@@ -251,9 +251,6 @@ export default {
             this.onRedraw();
         },
         onAddOutput(output, terminal) {
-            if (this.mapOver) {
-                terminal.setMapOver(this.mapOver);
-            }
             this.outputTerminals[output.name] = terminal;
             this.onRedraw();
         },
@@ -324,6 +321,7 @@ export default {
             this.inputs = data.inputs ? data.inputs.slice() : [];
             this.outputs = data.outputs ? data.outputs.slice() : [];
             const outputNames = this.outputs.map((output) => output.name);
+            this.activeOutputs.initialize(this.outputs, data.workflow_outputs);
             this.activeOutputs.filterOutputs(outputNames);
         },
         initData(data) {
@@ -332,7 +330,6 @@ export default {
             this.annotation = data.annotation;
             this.label = data.label;
             this.setData(data);
-            this.activeOutputs.initialize(this.outputs, data.workflow_outputs);
             this.showLoading = false;
         },
         labelOutput(outputName, label) {

client/src/components/Workflow/Editor/NodeInput.vue

@@ -103,7 +103,7 @@ export default {
             return terminal;
         },
         onChange() {
-            this.isMultiple = this.terminal.mapOver && this.terminal.mapOver.isCollection;
+            this.isMultiple = this.terminal.isMappedOver();
             this.$emit("onChange");
         },
         onRemove() {

client/src/components/Workflow/Editor/NodeOutput.vue

@@ -51,14 +51,11 @@ export default {
             if (Array.isArray(extensions)) {
                 extensions = extensions.join(", ");
             }
-            const activeLabel = this.activeOutput?.activeLabel || this.output.label || this.output.name;
+            const activeLabel = this.output.activeLabel || this.output.label || this.output.name;
             return `${activeLabel} (${extensions})`;
         },
-        activeOutput() {
-            return this.getNode().activeOutputs.outputsIndex[this.output.name];
-        },
         activeClass() {
-            return this.activeOutput?.activeOutput && "mark-terminal-active";
+            return this.output.activeOutput && "mark-terminal-active";
         },
         showCallout() {
             const node = this.getNode();
@@ -81,18 +78,17 @@ export default {
             } else {
                 // create new terminal, connect like old terminal, destroy old terminal
                 this.$emit("onRemove", this.output);
-                const newTerminal = this.createTerminal(newOutput);
-                newTerminal.connectors = this.terminal.connectors.map((c) => {
-                    return new Connector(this.getManager(), newTerminal, c.inputHandle);
+                this.createTerminal(newOutput);
+                this.terminal.connectors = oldTerminal.connectors.map((c) => {
+                    return new Connector(this.getManager(), this.terminal, c.inputHandle);
                 });
-                newTerminal.destroyInvalidConnections();
-                this.terminal = newTerminal;
+                this.terminal.destroyInvalidConnections();
                 oldTerminal.destroy();
             }
         },
     },
     mounted() {
-        this.terminal = this.createTerminal(this.output);
+        this.createTerminal(this.output);
     },
     methods: {
         terminalClassForOutput(output) {
@@ -105,7 +101,6 @@ export default {
             return terminalClass;
         },
         createTerminal(output) {
-            let terminal;
             const terminalClass = this.terminalClassForOutput(output);
             const parameters = {
                 node: this.getNode(),
@@ -116,33 +111,33 @@ export default {
             if (output.collection) {
                 const collection_type = output.collection_type;
                 const collection_type_source = output.collection_type_source;
-                terminal = new terminalClass({
+                this.terminal = new terminalClass({
                     ...parameters,
                     collection_type: collection_type,
                     collection_type_source: collection_type_source,
                     datatypes: output.extensions,
                 });
             } else if (output.parameter) {
-                terminal = new terminalClass({
+                this.terminal = new terminalClass({
                     ...parameters,
                     type: output.type,
                 });
             } else {
-                terminal = new terminalClass({
+                this.terminal = new terminalClass({
                     ...parameters,
                     datatypes: output.extensions,
                 });
             }
-            terminal.on("change", this.onChange.bind(this));
             new OutputDragging(this.getManager(), {
                 el: this.$refs.terminal,
-                terminal: terminal,
+                terminal: this.terminal,
             });
-            this.$emit("onAdd", this.output, terminal);
-            return terminal;
+            this.terminal.on("change", this.onChange.bind(this));
+            this.terminal.emit("change");
+            this.$emit("onAdd", this.output, this.terminal);
         },
         onChange() {
-            this.isMultiple = this.terminal.mapOver && this.terminal.mapOver.isCollection;
+            this.isMultiple = this.terminal.isMappedOver();
             this.$emit("onChange");
         },
         onToggle() {

client/src/components/Workflow/Editor/modules/outputs.js

@@ -127,6 +127,7 @@ export class ActiveOutputs {
         this.outputs &&
             this.outputs.forEach((o) => {
                 this.outputsIndex[o.name] = o;
+                this._updateOutput(o.name);
             });
     }
 }

client/src/components/Workflow/Editor/modules/terminals.js

@@ -169,6 +169,7 @@ class Terminal extends EventEmitter {
     }
     resetMapping() {
         this.mapOver = NULL_COLLECTION_TYPE_DESCRIPTION;
+        this.node.mapOver = undefined;
         this.emit("change");
     }
     resetCollectionTypeSource() {
@@ -590,6 +591,9 @@ class BaseOutputTerminal extends Terminal {
         super(attr);
         this.datatypes = attr.datatypes;
         this.optional = attr.optional;
+        if (this.node.mapOver) {
+            this.setMapOver(this.node.mapOver);
+        }
     }
     get force_datatype() {
         const changeOutputDatatype = this.node.postJobActions["ChangeDatatypeAction" + this.name];

client/src/components/plugins/legacyNavigation.js

@@ -36,7 +36,7 @@ export const legacyNavigationMixin = {
         // straight ifrme redirect
         iframeRedirect(path, target = "galaxy_main") {
             try {
-                const targetFrame = window.frames[target] || window.parent.frames[target];
+                const targetFrame = window.frames[target];
                 if (!targetFrame) {
                     throw new Error(`Requested frame ${target} doesn't exist`);
                 }

client/src/entry/analysis/AnalysisRouter.js

@@ -105,6 +105,8 @@ export const getAnalysisRouter = (Galaxy) => {
             "(/)collection(/)edit(/)(:collection_id)": "show_collection_edit_attributes",
             "(/)datasets/error": "show_dataset_error",
             "(/)datasets(/)(:dataset_id)/details": "show_dataset_details",
+            // legacy url for older links
+            "(/)datasets(/)(:dataset_id)/show_params": "show_dataset_details",
             "(/)interactivetool_entry_points(/)list": "show_interactivetool_list",
             "(/)libraries*path": "show_library_folder",
         },
@@ -123,7 +125,11 @@ export const getAnalysisRouter = (Galaxy) => {
             }
             this.page.display(container, noPadding);
             const mountFn = mountVueComponent(component);
-            return mountFn(propsData, container);
+            if (this.currentComponent && this.currentComponent.$destroy) {
+                this.currentComponent.$destroy();
+            }
+            this.currentComponent = mountFn(propsData, container);
+            return this.currentComponent;
         },
 
         show_tours: function (tour_id) {

client/src/layout/menu.js

@@ -90,25 +90,16 @@ export function fetchMenu(options = {}) {
     // Visualization tab.
     //
     if (Galaxy.config.visualizations_visible) {
-        menu.push({
-            id: "visualization",
-            title: _l("Visualize"),
-            url: "javascript:void(0)",
-            tooltip: _l("Visualize datasets"),
-            disabled: !Galaxy.user.id,
-            menu: [
-                {
-                    title: _l("Create Visualization"),
-                    url: "visualizations",
-                    target: "__use_router__",
-                },
-                {
-                    title: _l("Interactive Environments"),
-                    url: "visualization/gie_list",
-                    target: "galaxy_main",
-                },
-            ],
-        });
+        if (Galaxy.config.visualizations_visible) {
+            menu.push({
+                id: "visualization",
+                title: _l("Visualize"),
+                tooltip: _l("Visualize datasets"),
+                disabled: !Galaxy.user.id,
+                url: "visualizations",
+                target: "__use_router__",
+            });
+        }
     }
 
     //

client/src/mvc/dataset/dataset-li.js

@@ -9,6 +9,7 @@ import BASE_MVC from "mvc/base-mvc";
 import _l from "utils/localization";
 import { mountNametags } from "components/Nametags";
 import { Toast } from "ui/toast";
+import { getAppRoot } from "onload/loadConfig";
 
 var logNamespace = "dataset";
 /*==============================================================================
@@ -272,7 +273,8 @@ export var DatasetListItemView = _super.extend(
          *  @returns {jQuery} rendered DOM
          */
         _renderShowParamsButton: function () {
-            // gen. safe to show in all cases
+            const url = `datasets/${this.model.get("id")}/details`;
+
             return faIconButton({
                 title: _l("View details"),
                 classes: "params-btn",
@@ -281,9 +283,16 @@ export var DatasetListItemView = _super.extend(
                 faIcon: "fa-info-circle",
                 onclick: (ev) => {
                     const Galaxy = getGalaxyInstance();
-                    if (Galaxy.router) {
+                    if (Galaxy.frame && Galaxy.frame.active) {
                         ev.preventDefault();
-                        Galaxy.router.push(`/datasets/${this.model.get("id")}/details`);
+                        Galaxy.frame.add({
+                            url: `${getAppRoot()}${url}`,
+                            title: `Dataset Details of ${this.model.get("name")}`,
+                        });
+                    } else if (Galaxy.router) {
+                        ev.preventDefault();
+                        Galaxy.router.push(url);
+                        Galaxy.trigger("activate-hda", this.model.get("id"));
                     }
                 },
             });

client/src/mvc/form/form-data.js

@@ -125,10 +125,9 @@ export var Manager = Backbone.Model.extend({
     /** Matches a new tool model to the current input elements e.g. used to update dynamic options
      */
     matchModel: function (inputs, callback) {
-        var self = this;
         visitInputs(inputs, (input, name) => {
-            if (self.flat_dict[name]) {
-                callback(input, self.flat_dict[name]);
+            if (this.flat_dict[name]) {
+                callback(input, this.flat_dict[name]);
             }
         });
     },

client/src/style/scss/workflow.scss

@@ -113,12 +113,12 @@
             .input-terminal.multiple {
                 font-size: 1.2rem;
                 left: -0.8rem;
-                top: 2px;
+                top: 3px;
             }
             .output-terminal.multiple {
                 font-size: 1.2rem;
                 right: -0.8rem;
-                top: 2px;
+                bottom: 6px;
             }
             .input-terminal-active.can-accept > .icon {
                 color: $brand-success;

client/tests/qunit/tests/workflow_editor_tests.js

@@ -22,7 +22,7 @@ const createApp = function () {
 
 const nodeData = {
     inputs: [],
-    outputs: [],
+    outputs: [{ name: "out1", extensions: ["data"] }],
     config_form: "{}",
     tool_state: "ok",
     tool_errors: false,
@@ -326,6 +326,7 @@ QUnit.test("Collection output can connect to same collection input type", functi
     const outputTerminal = new Terminals.OutputCollectionTerminal({
         datatypes: "txt",
         collection_type: "list",
+        node: {},
     });
     outputTerminal.node = {postJobActions: {}};
     assert.ok(
@@ -340,6 +341,7 @@ QUnit.test("Optional collection output can not connect to required collection in
         datatypes: "txt",
         collection_type: "list",
         optional: true,
+        node: {},
     });
     outputTerminal.node = {};
     assert.ok(!inputTerminal.canAccept(outputTerminal).canAccept);
@@ -350,6 +352,7 @@ QUnit.test("Collection output cannot connect to different collection input type"
     const outputTerminal = new Terminals.OutputCollectionTerminal({
         datatypes: "txt",
         collection_type: "paired",
+        node: {},
     });
     outputTerminal.node = {};
     assert.ok(!inputTerminal.canAccept(outputTerminal).canAccept);
@@ -542,6 +545,7 @@ QUnit.module("Node view", {
                 datatypes: [outputType],
                 mapOver: Terminals.NULL_COLLECTION_TYPE_DESCRIPTION,
                 element: inputEl,
+                node: {},
             });
             outputTerminal.node = {
                 markChanged: function () {},
@@ -572,6 +576,7 @@ QUnit.module("Node view", {
                 datatypes: ["txt"],
                 mapOver: new Terminals.CollectionTypeDescription("list"),
                 element: inputEl,
+                node: {},
             });
             outputTerminal.node = {
                 markChanged: function () {},
@@ -598,6 +603,7 @@ QUnit.module("Node view", {
                 datatypes: ["txt"],
                 mapOver: new Terminals.CollectionTypeDescription("list"),
                 element: inputEl,
+                node: {},
             });
             outputTerminal.node = {
                 markChanged: function () {},
@@ -832,7 +838,10 @@ QUnit.test("equal", function (assert) {
 });
 
 QUnit.test("default constructor", function (assert) {
-    const terminal = new Terminals.InputTerminal({ input: {} });
+    const terminal = new Terminals.InputTerminal({
+        input: {},
+        node: {},
+    });
     assert.ok(terminal.mapOver === Terminals.NULL_COLLECTION_TYPE_DESCRIPTION);
 });
 
@@ -902,7 +911,11 @@ QUnit.module("terminal mapping logic", {
             output["extensions"] = ["data"];
         }
         const outputEl = $("<div>")[0];
-        const outputTerminal = new Terminals.OutputTerminal({ element: outputEl, datatypes: output.extensions });
+        const outputTerminal = new Terminals.OutputTerminal({
+            element: outputEl,
+            datatypes: output.extensions,
+            node: {},
+        });
         outputTerminal.node = node;
         if (mapOver) {
             outputTerminal.setMapOver(new Terminals.CollectionTypeDescription(mapOver));
@@ -921,6 +934,7 @@ QUnit.module("terminal mapping logic", {
             element: outputEl,
             datatypes: output.extensions,
             collection_type: collectionType,
+            node: {},
         });
         outputTerminal.node = node;
         if (mapOver) {
@@ -1265,3 +1279,14 @@ QUnit.test("simple mapping over collection outputs works correctly", function (a
     const testTerminal1 = this.newInputTerminal("list:list:list");
     this.verifyNotAttachable(assert, testTerminal1, connectedOutput);
 });
+
+QUnit.test("node mapping state over collection outputs works correctly", function (assert) {
+    const inputTerminal1 = this.newInputTerminal();
+    const outputCollectionTerminal1 = this.newOutputCollectionTerminal("list");
+    assert.ok(!inputTerminal1.node.mapOver);
+    const connector = new Connector({}, outputCollectionTerminal1, inputTerminal1);
+    outputCollectionTerminal1.connect(connector);
+    assert.ok(inputTerminal1.node.mapOver);
+    inputTerminal1.disconnect(connector);
+    assert.ok(!inputTerminal1.node.mapOver);
+});

config/plugins/webhooks/news/script.js

@@ -29,13 +29,11 @@ function addNewsIframe() {
     // now we'll hardcode the version users 'see'. @hexylena will remove this
     // code when she writes the user-facing release notes, and then will file
     // an issue for how we'll fix this properly.
-    if(currentGalaxyVersion == "22.01") {
+    if (currentGalaxyVersion == "22.01") {
         currentGalaxyVersion = "21.09";
-    } else if(currentGalaxyVersion == "21.09") {
-        currentGalaxyVersion = "21.05";
     }
 
-    const releaseNotes = `https://docs.galaxyproject.org/en/master/releases/${currentGalaxyVersion}_announce_user.html`;
+    const releaseNotes = `https://docs.galaxyproject.org/en/latest/releases/${currentGalaxyVersion}_announce_user.html`;
     const lastSeenVersion = window.localStorage.getItem("galaxy-news-seen-release");
     // Check that they've seen the current version's release notes.
     if (lastSeenVersion != currentGalaxyVersion) {

doc/source/admin/galaxy_options.rst

@@ -2788,18 +2788,16 @@
 :Type: str
 
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
-``sentry_sloreq_threshold``
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~
+``sentry_event_level``
+~~~~~~~~~~~~~~~~~~~~~~
 
 :Description:
-    Sentry slow request logging.  Requests slower than the threshold
-    indicated below will be sent as events to the configured Sentry
-    server (above, sentry_dsn).  A value of '0' is disabled.  For
-    example, you would set this to .005 to log all queries taking
-    longer than 5 milliseconds.
-:Default: ``0.0``
-:Type: float
+    Determines the minimum log level that will be sent as an event to
+    Sentry. Possible values are DEBUG, INFO, WARNING, ERROR or
+    CRITICAL.
+:Default: ``ERROR``
+:Type: str
 
 
 ~~~~~~~~~~~~~~~

doc/source/project/issues.rst

@@ -127,7 +127,7 @@ particular domain, as well as more organized release notes.
 - ``area/compliance``
 - ``area/configuration`` - Galaxy's configuration system
 - ``area/cwl`` - changes related to supporting the common workflow language in Galaxy
-- ``area/database`` - Change requires a modification to Galaxy's database
+- ``area/database`` - Change to Galaxy's database or data access layer
 - ``area/dataset-collections``
 - ``area/datatypes`` - Changes to Galaxy's datatypes
 - ``area/datatype-framework`` - Changes to Galaxy's datatype and metadata framework

doc/source/project/organization.rst

@@ -56,11 +56,13 @@ Members
 - Björn Grüning (@bgruening)
 - Aysam Guerler (@guerler)
 - Jennifer Hillman Jackson (@jennaj)
+- David López (@davelopez)
 - Anton Nekrutenko (@nekrut)
 - Helena Rasche (@hexylena)
 - Nicola Soranzo (@nsoranzo)
 - James Taylor (@jxtx)
 - Marius van den Beek (@mvdbeek)
+- Oleg Zharkov (@OlegZharkov)
 
 Membership
 ----------

doc/source/releases/21.09.rst

@@ -0,0 +1,681 @@
+
+.. to_doc
+
+21.09
+===============================
+
+.. announce_start
+
+Please note that the release is still being tested and finalized, and that these
+are pre-release notes.
+
+Check back here for the official and comprehensive notes soon!
+
+
+Enhancements
+-------------------------------
+
+.. major_feature
+
+
+.. feature
+
+* New User Welcome page
+  (thanks to `@astrovsky01 <https://github.com/astrovsky01>`__).
+  `Pull Request 11085`_
+* GPU enabled jupyter notebook for machine learning powered by Jupyter lab and Tensorflow
+  (thanks to `@anuprulez <https://github.com/anuprulez>`__).
+  `Pull Request 11484`_
+* NCBI Datasets data source tool
+  `Pull Request 11738`_
+* Feature/6965/change datatype and dbkey
+  (thanks to `@assuntad23 <https://github.com/assuntad23>`__).
+  `Pull Request 11799`_
+* Add "View Details" button for collections
+  `Pull Request 12261`_
+* Vuefy and improve granularity for tool HTML sanitization
+  `Pull Request 12283`_
+* Implement tool panel views.
+  `Pull Request 12327`_
+* Implement news webhook to handle release announcements
+  (thanks to `@hexylena <https://github.com/hexylena>`__).
+  `Pull Request 12396`_
+.. enhancement_tag_viz
+
+.. enhancement_tag_datatypes
+
+* Convert/standardize to fstrings across `lib`
+  `Pull Request 11338`_
+* Added documentation for FASTQ datatypes and implemented quality check
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 11931`_
+* Make dataset preview for h5mlm datatype
+  (thanks to `@qiagu <https://github.com/qiagu>`__).
+  `Pull Request 11935`_
+* Add datatypes for Structural Materials Hexrd application
+  (thanks to `@jj-umn <https://github.com/jj-umn>`__).
+  `Pull Request 11957`_
+* Adding new subclass types
+  (thanks to `@maikenp <https://github.com/maikenp>`__).
+  `Pull Request 12097`_
+* Converters: use target datatype
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12185`_
+* Add bref3 datatype
+  (thanks to `@gallardoalba <https://github.com/gallardoalba>`__).
+  `Pull Request 12199`_
+* Converters: add missing tests and add linting to converter tests
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12202`_
+* Converters: Unify converters to tabix and bgzip
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12213`_
+* Converters: Unify dcd, trr, xtc
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12224`_
+* converters: Unify bcf converters
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12225`_
+* Add "ExpressionSet RData object" Datatype
+  (thanks to `@mtekman <https://github.com/mtekman>`__).
+  `Pull Request 12336`_
+* also parse sam metadata from sam files
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12392`_
+* Add ONNX datatype
+  (thanks to `@anuprulez <https://github.com/anuprulez>`__).
+  `Pull Request 12429`_
+.. enhancement_tag_tools
+
+* Changes to Collection Operation Help sections and parameter names
+  `Pull Request 11068`_
+* Update bam.iobio interactive tool wrapper
+  (thanks to `@luke-c-sargent <https://github.com/luke-c-sargent>`__).
+  `Pull Request 11537`_
+* Add tool for exporting individual files to galaxy file source plugins.
+  `Pull Request 11613`_
+* Update AskOmics interactive tool to 4.3.1
+  (thanks to `@abretaud <https://github.com/abretaud>`__).
+  `Pull Request 12159`_
+* add <creator> to the tool schema template, use live links in xsd
+  `Pull Request 12242`_
+* Systematic handling of remotely required tool files.
+  `Pull Request 12250`_
+* Allow bio.tools mappings for legacy tools.
+  `Pull Request 12289`_
+* Allow skipping sanitization of / char in export_remote tool
+  `Pull Request 12372`_
+* Lock location file before adding values
+  `Pull Request 12446`_
+.. enhancement_tag_workflows
+
+.. enhancement_tag_ui
+
+* Beta history selenium updates and assorted bug fixes
+  `Pull Request 12351`_
+* webpack dev server on remote galaxy server
+  (thanks to `@OlegZharkov <https://github.com/OlegZharkov>`__).
+  `Pull Request 12356`_
+.. enhancement_tag_jobs
+
+* SGE DRMAA example for queue, RAM and CPUs request
+  (thanks to `@pcm32 <https://github.com/pcm32>`__).
+  `Pull Request 12059`_
+* Allow specifying useful classes of tools for mapping in job conf YAML/XML
+  `Pull Request 12258`_
+* Fix galaxy.json dynamic file collection in Pulsar.
+  `Pull Request 12267`_
+* Default k8s_unschedulable_walltime_limit to None and allow for unlimi…
+  (thanks to `@nuwang <https://github.com/nuwang>`__).
+  `Pull Request 12317`_
+* Support setting per-job-environment container resolvers using the `container_resolvers` key in the environment config
+  `Pull Request 12350`_
+* Add config options for controlling handler and runner monitor thread sleep.
+  `Pull Request 12473`_
+.. enhancement_tag_admin
+
+* Make conda_auto_init option overrridable
+  (thanks to `@rhpvorderman <https://github.com/rhpvorderman>`__).
+  `Pull Request 12200`_
+* Support setting the container resolvers configuration directly in the Galaxy app config
+  `Pull Request 12344`_
+* Improve k8s docker image
+  (thanks to `@nuwang <https://github.com/nuwang>`__).
+  `Pull Request 12373`_
+* update admin user list DiskUsageColumn sorting for null values
+  (thanks to `@cat-bro <https://github.com/cat-bro>`__).
+  `Pull Request 12469`_
+* Make the subject and body of the mailing list join message configurable
+  `Pull Request 12484`_
+.. enhancement
+
+* Test configuration for running Selenium tests against beta history panel.
+  `Pull Request 10965`_
+* Add `negate` attribute to validators .. and tests for all
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 11043`_
+* build list: list identifiers
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 11309`_
+* Prepare FastAPI route for quotas
+  (thanks to `@davelopez <https://github.com/davelopez>`__).
+  `Pull Request 11315`_
+* Create central pydantic models
+  (thanks to `@davelopez <https://github.com/davelopez>`__).
+  `Pull Request 11827`_
+* Use python:3.7-slim container instead of ubuntu
+  (thanks to `@rhpvorderman <https://github.com/rhpvorderman>`__).
+  `Pull Request 11855`_
+* add more examples for the file source plugins
+  (thanks to `@bgruening <https://github.com/bgruening>`__).
+  `Pull Request 11949`_
+* Migrate Libraries API (top level) to FastAPI
+  (thanks to `@davelopez <https://github.com/davelopez>`__).
+  `Pull Request 11983`_
+* Upgraded to v0.9.0 of irods client that supports parallel put/get
+  (thanks to `@kxk302 <https://github.com/kxk302>`__).
+  `Pull Request 11984`_
+* Improve handling of tags for collection manipulation tools
+  (thanks to `@simonbray <https://github.com/simonbray>`__).
+  `Pull Request 11992`_
+* Initial setup for migrating to declarative mapping
+  `Pull Request 12042`_
+* Migrate model to declarative mapping
+  `Pull Request 12045`_
+* Unify nested collection attribute or instance access
+  `Pull Request 12056`_
+* Increase API tests coverage for Data Library GET permissions
+  (thanks to `@davelopez <https://github.com/davelopez>`__).
+  `Pull Request 12057`_
+* Tweak PR template to be a bit more flexible and lightweight.
+  `Pull Request 12063`_
+* Declarative mappings + tests for all models
+  `Pull Request 12064`_
+* Enable pytest class-based tests
+  `Pull Request 12072`_
+* Rework variable name to clarify this is a wsgi application.
+  `Pull Request 12090`_
+* Upgraded irods client to v1.0.0
+  (thanks to `@kxk302 <https://github.com/kxk302>`__).
+  `Pull Request 12101`_
+* Add configuration option to hide user account management interfaces
+  `Pull Request 12103`_
+* Fix up some typing redundancy.
+  `Pull Request 12108`_
+* Rename dataset_collections_service to dataset_collection_manager.
+  `Pull Request 12110`_
+* allow jupyter IT to take multiple inputs
+  `Pull Request 12113`_
+* docs: change GIEs label to interactive-tools
+  `Pull Request 12114`_
+* Add FastAPI routes for Library Folders
+  (thanks to `@davelopez <https://github.com/davelopez>`__).
+  `Pull Request 12131`_
+* Change icon for "Share or Publish" in new history panel
+  (thanks to `@nsoranzo <https://github.com/nsoranzo>`__).
+  `Pull Request 12143`_
+* Cleanup use of refresh/undo/redo/sync icons
+  (thanks to `@nsoranzo <https://github.com/nsoranzo>`__).
+  `Pull Request 12145`_
+* Unify how the `GalaxyInteractorApi` handles JSON requests
+  (thanks to `@davelopez <https://github.com/davelopez>`__).
+  `Pull Request 12152`_
+* Hide masthead help resources if resource urls are unavailable
+  `Pull Request 12198`_
+* Cancel in progress CI on same branch, cache yarn cache
+  `Pull Request 12211`_
+* show target datatype in datatype conversion
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12230`_
+* Add a note to get Python extension and set interpreter
+  (thanks to `@eancelet <https://github.com/eancelet>`__).
+  `Pull Request 12234`_
+* Lint for whitespace in tool XML
+  (thanks to `@neoformit <https://github.com/neoformit>`__).
+  `Pull Request 12246`_
+* Use relative path in recursive discover_datasets tag
+  `Pull Request 12254`_
+* Deprecate data manager version
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12255`_
+* Add linting for tool parameter validators
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12262`_
+* Create a bug-report issue template
+  `Pull Request 12265`_
+* Enhance page functionalities in data-library folder
+  (thanks to `@OlegZharkov <https://github.com/OlegZharkov>`__).
+  `Pull Request 12270`_
+* Parse edam operation and topic data from bio.tools if available.
+  `Pull Request 12291`_
+* Refactor base toolbox code de-coupled from app & config into tool_util.
+  `Pull Request 12292`_
+* Vueify Dataset Error Details Page and Form
+  `Pull Request 12300`_
+* implement selecting a folder in file source (upload modal)
+  (thanks to `@OlegZharkov <https://github.com/OlegZharkov>`__).
+  `Pull Request 12310`_
+* allow HTML in DatasetStorage view
+  (thanks to `@bgruening <https://github.com/bgruening>`__).
+  `Pull Request 12312`_
+* xsd: document the delta attribute of has_size assert
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12314`_
+* make linter accept from_tool_provided_metadata as means to specify the format for discovered datasets
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12319`_
+* Add Collection Import Pages
+  `Pull Request 12324`_
+* Allow various configuration parameters to be set per host.
+  `Pull Request 12328`_
+* Migrate group_roles endpoints to fastapi
+  (thanks to `@ZimmerA <https://github.com/ZimmerA>`__).
+  `Pull Request 12337`_
+* Restore Upload dialog in global Galaxy object
+  `Pull Request 12339`_
+* Move page export button to title
+  `Pull Request 12340`_
+* API test case to ensure model attributes get escaped during tool evaluation
+  `Pull Request 12357`_
+* Provide more context in EDAM Tool Panel
+  `Pull Request 12365`_
+* fix verify output for collections
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12376`_
+* Extend per_host configurations to include static directories and files.
+  `Pull Request 12379`_
+* Display Tabular datasets as sortable tables, and PDF and HTML in embedded frames in Pages
+  `Pull Request 12387`_
+* create dirs when needed for the pyfilesystem export plugins
+  (thanks to `@bgruening <https://github.com/bgruening>`__).
+  `Pull Request 12388`_
+* Debug Galaxy (that uses a Slurm cluster, not a LocalRunner) in VSCode
+  (thanks to `@kxk302 <https://github.com/kxk302>`__).
+  `Pull Request 12403`_
+* Don't allow beta history panel Selenium tests to fail.
+  `Pull Request 12409`_
+* Selenium test for EDAM tool panel views.
+  `Pull Request 12422`_
+* Spanish Translations for remaining untranslated UI components
+  (thanks to `@hexylena <https://github.com/hexylena>`__).
+  `Pull Request 12427`_
+* Improvements to Selenium testing
+  `Pull Request 12433`_
+* Update refgenconf (refgenie) to 0.12.0. Requires version 0.4 of confi…
+  `Pull Request 12435`_
+* Set default constructor for SA models
+  `Pull Request 12438`_
+* FastAPI: add middelware to deal with file downloads
+  (thanks to `@davelopez <https://github.com/davelopez>`__).
+  `Pull Request 12441`_
+* Add options to override the history name and reuse histories in the tool test script
+  `Pull Request 12466`_
+* let invalid profile trigger linter error
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12501`_
+.. small_enhancement
+
+* Refactor Sharing
+  (thanks to `@OlegZharkov <https://github.com/OlegZharkov>`__).
+  `Pull Request 11701`_
+* Migrate to SQLAlchemy 1.4
+  `Pull Request 11737`_
+* Vueify Workflow Editor wrappers
+  `Pull Request 11898`_
+* add double quotes to cp for from_work_dir
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 11960`_
+* Prettier version update to new release
+  `Pull Request 12027`_
+* Vueify Tool Form Wrapper
+  `Pull Request 12031`_
+* Normalize Vue mounts in analysis, admin endpoints
+  `Pull Request 12040`_
+* Set cache_ok flag for SQLAlchemy custom types
+  `Pull Request 12060`_
+* Update SQLAlchemy to 1.4.17
+  `Pull Request 12089`_
+* Update dependency resolver docs and drop sample XML file in favor of direct configuration under the dependency_resolvers key of galaxy.yml
+  `Pull Request 12105`_
+* No change; triggering CI
+  `Pull Request 12111`_
+* Update SQLAlchemy to 1.4.18
+  `Pull Request 12132`_
+* Refactoring: Remove ExportsHistoryMixin and ImportsHistoryMixin
+  (thanks to `@davelopez <https://github.com/davelopez>`__).
+  `Pull Request 12146`_
+* Remove unused legacy controller things
+  `Pull Request 12172`_
+* Update SQLAlchmey to 1.4.19
+  `Pull Request 12188`_
+* Refactor Histories API controller
+  (thanks to `@davelopez <https://github.com/davelopez>`__).
+  `Pull Request 12195`_
+* Converters: Unify molecules converters
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12214`_
+* Vueify Workflow Run form
+  `Pull Request 12227`_
+* Update SQLAlchmey to 1.4.20
+  `Pull Request 12229`_
+* Refactor History contents API
+  (thanks to `@davelopez <https://github.com/davelopez>`__).
+  `Pull Request 12231`_
+* Improve `GitHubSearch` class
+  (thanks to `@nsoranzo <https://github.com/nsoranzo>`__).
+  `Pull Request 12276`_
+*  Some refactoring of interactive tool handling in k8s
+  (thanks to `@nuwang <https://github.com/nuwang>`__).
+  `Pull Request 12285`_
+* Replace form-view imports in Visualizations with Vue component layer
+  `Pull Request 12296`_
+* Fix histories content filtering by type
+  (thanks to `@davelopez <https://github.com/davelopez>`__).
+  `Pull Request 12303`_
+* Vueify Dataset Edit Attributes
+  `Pull Request 12323`_
+* Vueify Generic Form Wrapper
+  `Pull Request 12329`_
+* Refactor k8s IT checks to make logic more obvious
+  (thanks to `@nuwang <https://github.com/nuwang>`__).
+  `Pull Request 12360`_
+* Fix variable name typo in watchers
+  (thanks to `@nuwang <https://github.com/nuwang>`__).
+  `Pull Request 12368`_
+* FastAPI: deprecate url attribute (again)
+  (thanks to `@davelopez <https://github.com/davelopez>`__).
+  `Pull Request 12370`_
+* Avoid parsing the entire form model to the form display wrapper
+  `Pull Request 12374`_
+* Remove tool_conf.xml.main which is no longer used
+  (thanks to `@nuwang <https://github.com/nuwang>`__).
+  `Pull Request 12380`_
+* Removing unused poolsize from code and config.
+  (thanks to `@kxk302 <https://github.com/kxk302>`__).
+  `Pull Request 12381`_
+* FastAPI: support reverse URL lookups
+  (thanks to `@davelopez <https://github.com/davelopez>`__).
+  `Pull Request 12391`_
+* Adapt quotas tests to use JSON payload
+  (thanks to `@davelopez <https://github.com/davelopez>`__).
+  `Pull Request 12417`_
+* Declarative mapping follow-up: model constructors cleanup
+  `Pull Request 12424`_
+* Vuefy show_params.mako
+  (thanks to `@OlegZharkov <https://github.com/OlegZharkov>`__).
+  `Pull Request 12432`_
+* Return `tool_name` only when `show_job` is full
+  (thanks to `@OlegZharkov <https://github.com/OlegZharkov>`__).
+  `Pull Request 12457`_
+* Drop unused models: DeferredJob and TransferJob
+  `Pull Request 12461`_
+* Drop bcftools requirement from set_metadata tool
+  `Pull Request 12472`_
+* Revert "Install Tempita from a fork due to `use_2to3` deprecation"
+  (thanks to `@nsoranzo <https://github.com/nsoranzo>`__).
+  `Pull Request 12504`_
+* FastAPI: explicitly return empty response with 204 status
+  (thanks to `@davelopez <https://github.com/davelopez>`__).
+  `Pull Request 12505`_
+
+
+Fixes
+-------------------------------
+
+.. major_bug
+
+
+.. bug_tag_viz
+
+.. bug_tag_datatypes
+
+* Fix edta metadata setting
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12273`_
+* mothur datatype: PairwiseDistanceMatrix sniffer may be too unspecific
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12280`_
+* Small fix in binary.py
+  (thanks to `@melibleq <https://github.com/melibleq>`__).
+  `Pull Request 12384`_
+* Fix cmap sniffer
+  (thanks to `@astrovsky01 <https://github.com/astrovsky01>`__).
+  `Pull Request 12509`_
+.. bug_tag_tools
+
+* Fix typo in `interactivetool_jupyter_notebook.xml` help section
+  (thanks to `@maximskorik <https://github.com/maximskorik>`__).
+  `Pull Request 12077`_
+* Update interactivetool_pyiron.xml
+  (thanks to `@gmauro <https://github.com/gmauro>`__).
+  `Pull Request 12127`_
+* Fix patterns in Grep1 tool
+  `Pull Request 12166`_
+* Restore Grep1 version 1.0.1
+  `Pull Request 12252`_
+.. bug_tag_workflows
+
+* Add switch for optional select in workflow editor
+  `Pull Request 12138`_
+* Do not persist tool state when invoking workflow
+  `Pull Request 12141`_
+* Fix failure to serialize invocation / drop stored_workflow_id from invocation API
+  `Pull Request 12148`_
+* Fix various TRS bugs and add tests
+  `Pull Request 12182`_
+* Fix simplified workflow run form config setting
+  `Pull Request 12204`_
+* Fix expanding collections in invocation view
+  `Pull Request 12216`_
+.. bug_tag_ui
+
+* Swap to galaxy fork of bootstrap-tour
+  `Pull Request 12210`_
+* Beta history annotation, title, and tag editor
+  `Pull Request 12299`_
+* Fix typo in docs
+  `Pull Request 12450`_
+.. bug_tag_jobs
+
+* Handle k8s job creation failures
+  (thanks to `@brinkmanlab <https://github.com/brinkmanlab>`__).
+  `Pull Request 11715`_
+* More IT and k8s runner fixes
+  (thanks to `@almahmoud <https://github.com/almahmoud>`__).
+  `Pull Request 11989`_
+* Unquote globs in from_work_dir ouptuts
+  `Pull Request 12049`_
+* quote extract_dataset_parts.sh call
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12183`_
+* Fix crash when uwsgi + mules has unhandled jobs at startup
+  `Pull Request 12215`_
+* k8s - handle metadata explicitly instead of relying on legacy fallback
+  (thanks to `@nuwang <https://github.com/nuwang>`__).
+  `Pull Request 12243`_
+* Fix version command when using extended metadata
+  `Pull Request 12245`_
+* Fix composite inputs when using Pulsar.
+  `Pull Request 12259`_
+* Fix Pulsar dynamic output discovery for various simple tool specified patterns.
+  `Pull Request 12260`_
+* Added additional guard conditions in k8s runner to prevent KeyError
+  (thanks to `@nuwang <https://github.com/nuwang>`__).
+  `Pull Request 12277`_
+* Fix Pulsar's ability to read tool provided metadata when path is overridden.
+  `Pull Request 12284`_
+* Continue handler startup even if job working directory can't be recovered
+  `Pull Request 12316`_
+* tmp_dir can be an expression or the string True - fix sample job_conf…
+  (thanks to `@nuwang <https://github.com/nuwang>`__).
+  `Pull Request 12361`_
+* Fail data manager job when processing table entry fails
+  `Pull Request 12495`_
+.. bug_tag_admin
+
+* change doc for ftp_upload_dir/site
+  (thanks to `@lldelisle <https://github.com/lldelisle>`__).
+  `Pull Request 12164`_
+.. bug
+
+* Fix non-closing masthread downdown
+  (thanks to `@OlegZharkov <https://github.com/OlegZharkov>`__).
+  `Pull Request 11962`_
+* Update tensorflow to 2.5.0.
+  `Pull Request 12026`_
+* Fix query parameter in `/api/libraries/{id}/permissions`
+  (thanks to `@davelopez <https://github.com/davelopez>`__).
+  `Pull Request 12047`_
+* Add/drop index together with `job_id` column
+  (thanks to `@nsoranzo <https://github.com/nsoranzo>`__).
+  `Pull Request 12050`_
+* Fix pysam.view call
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12061`_
+* Open partfile in rb-mode in multipart uploading
+  (thanks to `@bioflowy <https://github.com/bioflowy>`__).
+  `Pull Request 12084`_
+* Make components full width
+  (thanks to `@hexylena <https://github.com/hexylena>`__).
+  `Pull Request 12085`_
+* Fix history audit table row trigger
+  `Pull Request 12130`_
+* Update fastapi to 0.65.2 and dependencies
+  (thanks to `@davelopez <https://github.com/davelopez>`__).
+  `Pull Request 12136`_
+* Fix subworkflow tool state upgrade parsing
+  `Pull Request 12139`_
+* adding test case for pair name, fixing other listpaired test case
+  (thanks to `@assuntad23 <https://github.com/assuntad23>`__).
+  `Pull Request 12154`_
+* Fix remote files upload from other pages
+  (thanks to `@almahmoud <https://github.com/almahmoud>`__).
+  `Pull Request 12155`_
+* Add timeout to all urlopen and requests.* methods
+  `Pull Request 12173`_
+* Allow ``decompress="true"`` in test comparison methods
+  `Pull Request 12187`_
+* Fix TRS search if search term contains -
+  `Pull Request 12196`_
+* Allow leading whitespace in TRS search term
+  `Pull Request 12197`_
+* xsd: fix typo
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12206`_
+* Work around null hid
+  `Pull Request 12217`_
+* Assorted fixes to debugging documentation
+  `Pull Request 12218`_
+* Don't let reports auto-create database tables
+  `Pull Request 12220`_
+* Small fixes for beta history panel.
+  `Pull Request 12228`_
+* Fix fugue pencil on old sharing not showing up correctly.
+  `Pull Request 12235`_
+* Fix string formatting bug in test interactor
+  `Pull Request 12253`_
+* Added check to install 'fs.sshfs' package
+  (thanks to `@kjetilkl <https://github.com/kjetilkl>`__).
+  `Pull Request 12269`_
+* fix type of  `page` and `page_limit` in Libraries Folder permissions
+  (thanks to `@OlegZharkov <https://github.com/OlegZharkov>`__).
+  `Pull Request 12271`_
+* Improve handling of unexpected values in is_empty_file
+  `Pull Request 12275`_
+* Beta history: Dataset Collection API fix for updating tags
+  `Pull Request 12286`_
+* Allow scrolling in autocomplete options, fix search word filtering
+  `Pull Request 12290`_
+* Add a GDPR-friendly user purge to pgcleanup
+  `Pull Request 12293`_
+* Fix admin user manage
+  `Pull Request 12295`_
+* reword history import button text
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12304`_
+* Fix order of dataset instances
+  `Pull Request 12305`_
+* Fix recursive file list in posix remote files
+  (thanks to `@OlegZharkov <https://github.com/OlegZharkov>`__).
+  `Pull Request 12307`_
+* linter support for `format_source` in static collection elements
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12309`_
+* Make authnz end session endpoint optional to support various …
+  (thanks to `@nuwang <https://github.com/nuwang>`__).
+  `Pull Request 12311`_
+* Use workflow step title as defined in the workflow model
+  `Pull Request 12320`_
+* Gitlab Error report - Remove previously required urllib
+  (thanks to `@selten <https://github.com/selten>`__).
+  `Pull Request 12333`_
+* Identify tool by name, owner, toolshed
+  (thanks to `@pvanheus <https://github.com/pvanheus>`__).
+  `Pull Request 12338`_
+* Fix for empty docker_volumes/singularity_volumes in job config
+  `Pull Request 12341`_
+* Fix up plugin_config module modifying configuration dictionaries.
+  `Pull Request 12354`_
+* Remove NetworkException catch clauses, so if irods server is down, NetworkException percolates up, preventing a silent failure and alerting us
+  (thanks to `@kxk302 <https://github.com/kxk302>`__).
+  `Pull Request 12363`_
+* Bump tensorflow from 2.5.0 to 2.5.1 in /lib/galaxy/dependencies
+  `Pull Request 12378`_
+* Added missing triggers package to galaxy data
+  (thanks to `@nuwang <https://github.com/nuwang>`__).
+  `Pull Request 12389`_
+* Fix hids, alignment, other style issues in beta history.
+  `Pull Request 12406`_
+* Fixes to selenium testing with beta history panel enabled.
+  `Pull Request 12407`_
+* Downgrade Cheetah3 to 3.2.6.post1
+  (thanks to `@nsoranzo <https://github.com/nsoranzo>`__).
+  `Pull Request 12408`_
+* Install Tempita from a fork due to `use_2to3` deprecation
+  (thanks to `@nsoranzo <https://github.com/nsoranzo>`__).
+  `Pull Request 12413`_
+* Fix flaky failing workflow editor selenium test.
+  `Pull Request 12423`_
+* Bump sqlparse from 0.4.1 to 0.4.2 in /lib/galaxy/dependencies
+  `Pull Request 12436`_
+* Fix keycloak user creation username handling.
+  `Pull Request 12444`_
+* Make Role description field optional
+  (thanks to `@davelopez <https://github.com/davelopez>`__).
+  `Pull Request 12447`_
+* Drop sylink to tool_conf.xml.main
+  `Pull Request 12451`_
+* Fix history contents API update batch with collections
+  (thanks to `@davelopez <https://github.com/davelopez>`__).
+  `Pull Request 12456`_
+* Remove default pagination in `LibraryPopulator.get_permissions()`
+  (thanks to `@davelopez <https://github.com/davelopez>`__).
+  `Pull Request 12463`_
+* Drop tool_name from dataset details
+  `Pull Request 12465`_
+* stripping whitespace on urls when uploading
+  (thanks to `@assuntad23 <https://github.com/assuntad23>`__).
+  `Pull Request 12474`_
+* removing appended name field for default_value
+  (thanks to `@assuntad23 <https://github.com/assuntad23>`__).
+  `Pull Request 12481`_
+* Explicitly set `page_limit` in `test_get_library_available_permissions`
+  (thanks to `@davelopez <https://github.com/davelopez>`__).
+  `Pull Request 12483`_
+* Update dataset edit toggle button to refer to dataset instead of coll…
+  `Pull Request 12485`_
+* Handle deleted panel views in the UI.
+  `Pull Request 12488`_
+* Fix bug when restarting Galaxy in selenium integration tests.
+  `Pull Request 12489`_
+* revert a change to xsd
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12494`_
+* Add PostgreSQL < 9.5 deprecation notice to 21.05 release announce
+  (thanks to `@nsoranzo <https://github.com/nsoranzo>`__).
+  `Pull Request 12496`_
+* Fix indention for admin panel links
+  `Pull Request 12497`_
+
+.. include:: 21.09_prs.rst
+

doc/source/releases/21.09_announce.rst

@@ -0,0 +1,50 @@
+
+===========================================================
+September 2021 Galaxy Release (v 21.09)
+===========================================================
+
+.. include:: _header.rst
+
+Highlights
+===========================================================
+
+Please note that the release is still being tested and finalized, and that these
+are pre-release notes.
+
+Check back here for the official and comprehensive notes soon!
+
+**Feature1**
+  Feature description.
+
+**Feature2**
+  Feature description.
+
+**Feature3**
+  Feature description.
+
+Also check out the `21.09 user release notes <21.09_announce_user.html>`__
+
+Get Galaxy
+==========
+
+The code lives at `GitHub <https://github.com/galaxyproject/galaxy>`__ and you should have `Git <https://git-scm.com/>`__ to obtain it.
+
+To get a new Galaxy repository run:
+  .. code-block:: shell
+
+      $ git clone -b release_21.09 https://github.com/galaxyproject/galaxy.git
+
+To update an existing Galaxy repository run:
+  .. code-block:: shell
+
+      $ git fetch origin && git checkout release_21.09 && git pull --ff-only origin release_21.09
+
+See the `community hub <https://galaxyproject.org/develop/source-code/>`__ for additional details on source code locations.
+
+Release Notes
+===========================================================
+
+.. include:: 21.09.rst
+   :start-after: announce_start
+
+.. include:: _thanks.rst

doc/source/releases/21.09_announce_user.rst

@@ -0,0 +1,152 @@
+
+===========================================================
+September 2021 Galaxy Release (v 21.09)
+===========================================================
+
+.. include:: _header.rst
+
+Highlights
+===========================================================
+
+Please note that the release is still being tested and finalized, and that these
+are pre-release notes.
+
+Check back here for the official and comprehensive notes soon!
+
+**Feature1**
+  Feature description.
+
+**Feature2**
+  Feature description.
+
+**Feature3**
+  Feature description.
+
+
+New Visualizations
+===========================================================
+
+.. visualizations
+
+New Datatypes
+===========================================================
+
+.. datatypes
+* Convert/standardize to fstrings across `lib`
+  `Pull Request 11338`_
+* Added documentation for FASTQ datatypes and implemented quality check
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 11931`_
+* Make dataset preview for h5mlm datatype
+  (thanks to `@qiagu <https://github.com/qiagu>`__).
+  `Pull Request 11935`_
+* Add datatypes for Structural Materials Hexrd application
+  (thanks to `@jj-umn <https://github.com/jj-umn>`__).
+  `Pull Request 11957`_
+* Adding new subclass types
+  (thanks to `@maikenp <https://github.com/maikenp>`__).
+  `Pull Request 12097`_
+* Converters: use target datatype
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12185`_
+* Add bref3 datatype
+  (thanks to `@gallardoalba <https://github.com/gallardoalba>`__).
+  `Pull Request 12199`_
+* Converters: add missing tests and add linting to converter tests
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12202`_
+* Converters: Unify converters to tabix and bgzip
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12213`_
+* Converters: Unify molecules converters
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12214`_
+* Converters: Unify dcd, trr, xtc
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12224`_
+* converters: Unify bcf converters
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12225`_
+* Fix edta metadata setting
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12273`_
+* mothur datatype: PairwiseDistanceMatrix sniffer may be too unspecific
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12280`_
+* Add "ExpressionSet RData object" Datatype
+  (thanks to `@mtekman <https://github.com/mtekman>`__).
+  `Pull Request 12336`_
+* Small fix in binary.py
+  (thanks to `@melibleq <https://github.com/melibleq>`__).
+  `Pull Request 12384`_
+* also parse sam metadata from sam files
+  (thanks to `@bernt-matthias <https://github.com/bernt-matthias>`__).
+  `Pull Request 12392`_
+* Add ONNX datatype
+  (thanks to `@anuprulez <https://github.com/anuprulez>`__).
+  `Pull Request 12429`_
+* Drop bcftools requirement from set_metadata tool
+  `Pull Request 12472`_
+* Fix cmap sniffer
+  (thanks to `@astrovsky01 <https://github.com/astrovsky01>`__).
+  `Pull Request 12509`_
+
+Builtin Tool Updates
+===========================================================
+
+.. tools
+* Changes to Collection Operation Help sections and parameter names
+  `Pull Request 11068`_
+* GPU enabled jupyter notebook for machine learning powered by Jupyter lab and Tensorflow
+  (thanks to `@anuprulez <https://github.com/anuprulez>`__).
+  `Pull Request 11484`_
+* Update bam.iobio interactive tool wrapper
+  (thanks to `@luke-c-sargent <https://github.com/luke-c-sargent>`__).
+  `Pull Request 11537`_
+* Add tool for exporting individual files to galaxy file source plugins.
+  `Pull Request 11613`_
+* NCBI Datasets data source tool
+  `Pull Request 11738`_
+* Fix typo in `interactivetool_jupyter_notebook.xml` help section
+  (thanks to `@maximskorik <https://github.com/maximskorik>`__).
+  `Pull Request 12077`_
+* Update interactivetool_pyiron.xml
+  (thanks to `@gmauro <https://github.com/gmauro>`__).
+  `Pull Request 12127`_
+* Update AskOmics interactive tool to 4.3.1
+  (thanks to `@abretaud <https://github.com/abretaud>`__).
+  `Pull Request 12159`_
+* Fix patterns in Grep1 tool
+  `Pull Request 12166`_
+* Remove unused legacy controller things
+  `Pull Request 12172`_
+* add <creator> to the tool schema template, use live links in xsd
+  `Pull Request 12242`_
+* Systematic handling of remotely required tool files.
+  `Pull Request 12250`_
+* Restore Grep1 version 1.0.1
+  `Pull Request 12252`_
+* Vuefy and improve granularity for tool HTML sanitization
+  `Pull Request 12283`_
+* Allow bio.tools mappings for legacy tools.
+  `Pull Request 12289`_
+* Allow skipping sanitization of / char in export_remote tool
+  `Pull Request 12372`_
+* Lock location file before adding values
+  `Pull Request 12446`_
+
+Release Testing Team
+===========================================================
+
+A special thanks to the release testing team for testing many of the new features and reporting many bugs:
+
+<team members go here>
+
+Release Notes
+===========================================================
+
+Please see the `full release notes <21.09_announce.html>`_ for more details.
+
+.. include:: 21.09_prs.rst
+
+.. include:: _thanks.rst

doc/source/releases/21.09_prs.rst

@@ -0,0 +1,269 @@
+
+.. github_links
+.. _Pull Request 10965: https://github.com/galaxyproject/galaxy/pull/10965
+.. _Pull Request 11043: https://github.com/galaxyproject/galaxy/pull/11043
+.. _Pull Request 11068: https://github.com/galaxyproject/galaxy/pull/11068
+.. _Pull Request 11085: https://github.com/galaxyproject/galaxy/pull/11085
+.. _Pull Request 11309: https://github.com/galaxyproject/galaxy/pull/11309
+.. _Pull Request 11315: https://github.com/galaxyproject/galaxy/pull/11315
+.. _Pull Request 11338: https://github.com/galaxyproject/galaxy/pull/11338
+.. _Pull Request 11484: https://github.com/galaxyproject/galaxy/pull/11484
+.. _Pull Request 11537: https://github.com/galaxyproject/galaxy/pull/11537
+.. _Pull Request 11613: https://github.com/galaxyproject/galaxy/pull/11613
+.. _Pull Request 11701: https://github.com/galaxyproject/galaxy/pull/11701
+.. _Pull Request 11715: https://github.com/galaxyproject/galaxy/pull/11715
+.. _Pull Request 11737: https://github.com/galaxyproject/galaxy/pull/11737
+.. _Pull Request 11738: https://github.com/galaxyproject/galaxy/pull/11738
+.. _Pull Request 11799: https://github.com/galaxyproject/galaxy/pull/11799
+.. _Pull Request 11827: https://github.com/galaxyproject/galaxy/pull/11827
+.. _Pull Request 11855: https://github.com/galaxyproject/galaxy/pull/11855
+.. _Pull Request 11898: https://github.com/galaxyproject/galaxy/pull/11898
+.. _Pull Request 11931: https://github.com/galaxyproject/galaxy/pull/11931
+.. _Pull Request 11935: https://github.com/galaxyproject/galaxy/pull/11935
+.. _Pull Request 11937: https://github.com/galaxyproject/galaxy/pull/11937
+.. _Pull Request 11949: https://github.com/galaxyproject/galaxy/pull/11949
+.. _Pull Request 11957: https://github.com/galaxyproject/galaxy/pull/11957
+.. _Pull Request 11960: https://github.com/galaxyproject/galaxy/pull/11960
+.. _Pull Request 11962: https://github.com/galaxyproject/galaxy/pull/11962
+.. _Pull Request 11983: https://github.com/galaxyproject/galaxy/pull/11983
+.. _Pull Request 11984: https://github.com/galaxyproject/galaxy/pull/11984
+.. _Pull Request 11989: https://github.com/galaxyproject/galaxy/pull/11989
+.. _Pull Request 11992: https://github.com/galaxyproject/galaxy/pull/11992
+.. _Pull Request 12026: https://github.com/galaxyproject/galaxy/pull/12026
+.. _Pull Request 12027: https://github.com/galaxyproject/galaxy/pull/12027
+.. _Pull Request 12031: https://github.com/galaxyproject/galaxy/pull/12031
+.. _Pull Request 12040: https://github.com/galaxyproject/galaxy/pull/12040
+.. _Pull Request 12042: https://github.com/galaxyproject/galaxy/pull/12042
+.. _Pull Request 12045: https://github.com/galaxyproject/galaxy/pull/12045
+.. _Pull Request 12047: https://github.com/galaxyproject/galaxy/pull/12047
+.. _Pull Request 12049: https://github.com/galaxyproject/galaxy/pull/12049
+.. _Pull Request 12050: https://github.com/galaxyproject/galaxy/pull/12050
+.. _Pull Request 12056: https://github.com/galaxyproject/galaxy/pull/12056
+.. _Pull Request 12057: https://github.com/galaxyproject/galaxy/pull/12057
+.. _Pull Request 12059: https://github.com/galaxyproject/galaxy/pull/12059
+.. _Pull Request 12060: https://github.com/galaxyproject/galaxy/pull/12060
+.. _Pull Request 12061: https://github.com/galaxyproject/galaxy/pull/12061
+.. _Pull Request 12063: https://github.com/galaxyproject/galaxy/pull/12063
+.. _Pull Request 12064: https://github.com/galaxyproject/galaxy/pull/12064
+.. _Pull Request 12072: https://github.com/galaxyproject/galaxy/pull/12072
+.. _Pull Request 12077: https://github.com/galaxyproject/galaxy/pull/12077
+.. _Pull Request 12084: https://github.com/galaxyproject/galaxy/pull/12084
+.. _Pull Request 12085: https://github.com/galaxyproject/galaxy/pull/12085
+.. _Pull Request 12089: https://github.com/galaxyproject/galaxy/pull/12089
+.. _Pull Request 12090: https://github.com/galaxyproject/galaxy/pull/12090
+.. _Pull Request 12097: https://github.com/galaxyproject/galaxy/pull/12097
+.. _Pull Request 12101: https://github.com/galaxyproject/galaxy/pull/12101
+.. _Pull Request 12103: https://github.com/galaxyproject/galaxy/pull/12103
+.. _Pull Request 12105: https://github.com/galaxyproject/galaxy/pull/12105
+.. _Pull Request 12108: https://github.com/galaxyproject/galaxy/pull/12108
+.. _Pull Request 12110: https://github.com/galaxyproject/galaxy/pull/12110
+.. _Pull Request 12111: https://github.com/galaxyproject/galaxy/pull/12111
+.. _Pull Request 12113: https://github.com/galaxyproject/galaxy/pull/12113
+.. _Pull Request 12114: https://github.com/galaxyproject/galaxy/pull/12114
+.. _Pull Request 12127: https://github.com/galaxyproject/galaxy/pull/12127
+.. _Pull Request 12130: https://github.com/galaxyproject/galaxy/pull/12130
+.. _Pull Request 12131: https://github.com/galaxyproject/galaxy/pull/12131
+.. _Pull Request 12132: https://github.com/galaxyproject/galaxy/pull/12132
+.. _Pull Request 12136: https://github.com/galaxyproject/galaxy/pull/12136
+.. _Pull Request 12138: https://github.com/galaxyproject/galaxy/pull/12138
+.. _Pull Request 12139: https://github.com/galaxyproject/galaxy/pull/12139
+.. _Pull Request 12140: https://github.com/galaxyproject/galaxy/pull/12140
+.. _Pull Request 12141: https://github.com/galaxyproject/galaxy/pull/12141
+.. _Pull Request 12143: https://github.com/galaxyproject/galaxy/pull/12143
+.. _Pull Request 12145: https://github.com/galaxyproject/galaxy/pull/12145
+.. _Pull Request 12146: https://github.com/galaxyproject/galaxy/pull/12146
+.. _Pull Request 12148: https://github.com/galaxyproject/galaxy/pull/12148
+.. _Pull Request 12152: https://github.com/galaxyproject/galaxy/pull/12152
+.. _Pull Request 12154: https://github.com/galaxyproject/galaxy/pull/12154
+.. _Pull Request 12155: https://github.com/galaxyproject/galaxy/pull/12155
+.. _Pull Request 12159: https://github.com/galaxyproject/galaxy/pull/12159
+.. _Pull Request 12164: https://github.com/galaxyproject/galaxy/pull/12164
+.. _Pull Request 12166: https://github.com/galaxyproject/galaxy/pull/12166
+.. _Pull Request 12172: https://github.com/galaxyproject/galaxy/pull/12172
+.. _Pull Request 12173: https://github.com/galaxyproject/galaxy/pull/12173
+.. _Pull Request 12182: https://github.com/galaxyproject/galaxy/pull/12182
+.. _Pull Request 12183: https://github.com/galaxyproject/galaxy/pull/12183
+.. _Pull Request 12185: https://github.com/galaxyproject/galaxy/pull/12185
+.. _Pull Request 12187: https://github.com/galaxyproject/galaxy/pull/12187
+.. _Pull Request 12188: https://github.com/galaxyproject/galaxy/pull/12188
+.. _Pull Request 12195: https://github.com/galaxyproject/galaxy/pull/12195
+.. _Pull Request 12196: https://github.com/galaxyproject/galaxy/pull/12196
+.. _Pull Request 12197: https://github.com/galaxyproject/galaxy/pull/12197
+.. _Pull Request 12198: https://github.com/galaxyproject/galaxy/pull/12198
+.. _Pull Request 12199: https://github.com/galaxyproject/galaxy/pull/12199
+.. _Pull Request 12200: https://github.com/galaxyproject/galaxy/pull/12200
+.. _Pull Request 12202: https://github.com/galaxyproject/galaxy/pull/12202
+.. _Pull Request 12204: https://github.com/galaxyproject/galaxy/pull/12204
+.. _Pull Request 12206: https://github.com/galaxyproject/galaxy/pull/12206
+.. _Pull Request 12210: https://github.com/galaxyproject/galaxy/pull/12210
+.. _Pull Request 12211: https://github.com/galaxyproject/galaxy/pull/12211
+.. _Pull Request 12213: https://github.com/galaxyproject/galaxy/pull/12213
+.. _Pull Request 12214: https://github.com/galaxyproject/galaxy/pull/12214
+.. _Pull Request 12215: https://github.com/galaxyproject/galaxy/pull/12215
+.. _Pull Request 12216: https://github.com/galaxyproject/galaxy/pull/12216
+.. _Pull Request 12217: https://github.com/galaxyproject/galaxy/pull/12217
+.. _Pull Request 12218: https://github.com/galaxyproject/galaxy/pull/12218
+.. _Pull Request 12220: https://github.com/galaxyproject/galaxy/pull/12220
+.. _Pull Request 12221: https://github.com/galaxyproject/galaxy/pull/12221
+.. _Pull Request 12224: https://github.com/galaxyproject/galaxy/pull/12224
+.. _Pull Request 12225: https://github.com/galaxyproject/galaxy/pull/12225
+.. _Pull Request 12227: https://github.com/galaxyproject/galaxy/pull/12227
+.. _Pull Request 12228: https://github.com/galaxyproject/galaxy/pull/12228
+.. _Pull Request 12229: https://github.com/galaxyproject/galaxy/pull/12229
+.. _Pull Request 12230: https://github.com/galaxyproject/galaxy/pull/12230
+.. _Pull Request 12231: https://github.com/galaxyproject/galaxy/pull/12231
+.. _Pull Request 12234: https://github.com/galaxyproject/galaxy/pull/12234
+.. _Pull Request 12235: https://github.com/galaxyproject/galaxy/pull/12235
+.. _Pull Request 12242: https://github.com/galaxyproject/galaxy/pull/12242
+.. _Pull Request 12243: https://github.com/galaxyproject/galaxy/pull/12243
+.. _Pull Request 12245: https://github.com/galaxyproject/galaxy/pull/12245
+.. _Pull Request 12246: https://github.com/galaxyproject/galaxy/pull/12246
+.. _Pull Request 12250: https://github.com/galaxyproject/galaxy/pull/12250
+.. _Pull Request 12252: https://github.com/galaxyproject/galaxy/pull/12252
+.. _Pull Request 12253: https://github.com/galaxyproject/galaxy/pull/12253
+.. _Pull Request 12254: https://github.com/galaxyproject/galaxy/pull/12254
+.. _Pull Request 12255: https://github.com/galaxyproject/galaxy/pull/12255
+.. _Pull Request 12258: https://github.com/galaxyproject/galaxy/pull/12258
+.. _Pull Request 12259: https://github.com/galaxyproject/galaxy/pull/12259
+.. _Pull Request 12260: https://github.com/galaxyproject/galaxy/pull/12260
+.. _Pull Request 12261: https://github.com/galaxyproject/galaxy/pull/12261
+.. _Pull Request 12262: https://github.com/galaxyproject/galaxy/pull/12262
+.. _Pull Request 12265: https://github.com/galaxyproject/galaxy/pull/12265
+.. _Pull Request 12266: https://github.com/galaxyproject/galaxy/pull/12266
+.. _Pull Request 12267: https://github.com/galaxyproject/galaxy/pull/12267
+.. _Pull Request 12269: https://github.com/galaxyproject/galaxy/pull/12269
+.. _Pull Request 12270: https://github.com/galaxyproject/galaxy/pull/12270
+.. _Pull Request 12271: https://github.com/galaxyproject/galaxy/pull/12271
+.. _Pull Request 12273: https://github.com/galaxyproject/galaxy/pull/12273
+.. _Pull Request 12275: https://github.com/galaxyproject/galaxy/pull/12275
+.. _Pull Request 12276: https://github.com/galaxyproject/galaxy/pull/12276
+.. _Pull Request 12277: https://github.com/galaxyproject/galaxy/pull/12277
+.. _Pull Request 12280: https://github.com/galaxyproject/galaxy/pull/12280
+.. _Pull Request 12283: https://github.com/galaxyproject/galaxy/pull/12283
+.. _Pull Request 12284: https://github.com/galaxyproject/galaxy/pull/12284
+.. _Pull Request 12285: https://github.com/galaxyproject/galaxy/pull/12285
+.. _Pull Request 12286: https://github.com/galaxyproject/galaxy/pull/12286
+.. _Pull Request 12289: https://github.com/galaxyproject/galaxy/pull/12289
+.. _Pull Request 12290: https://github.com/galaxyproject/galaxy/pull/12290
+.. _Pull Request 12291: https://github.com/galaxyproject/galaxy/pull/12291
+.. _Pull Request 12292: https://github.com/galaxyproject/galaxy/pull/12292
+.. _Pull Request 12293: https://github.com/galaxyproject/galaxy/pull/12293
+.. _Pull Request 12295: https://github.com/galaxyproject/galaxy/pull/12295
+.. _Pull Request 12296: https://github.com/galaxyproject/galaxy/pull/12296
+.. _Pull Request 12299: https://github.com/galaxyproject/galaxy/pull/12299
+.. _Pull Request 12300: https://github.com/galaxyproject/galaxy/pull/12300
+.. _Pull Request 12303: https://github.com/galaxyproject/galaxy/pull/12303
+.. _Pull Request 12304: https://github.com/galaxyproject/galaxy/pull/12304
+.. _Pull Request 12305: https://github.com/galaxyproject/galaxy/pull/12305
+.. _Pull Request 12307: https://github.com/galaxyproject/galaxy/pull/12307
+.. _Pull Request 12309: https://github.com/galaxyproject/galaxy/pull/12309
+.. _Pull Request 12310: https://github.com/galaxyproject/galaxy/pull/12310
+.. _Pull Request 12311: https://github.com/galaxyproject/galaxy/pull/12311
+.. _Pull Request 12312: https://github.com/galaxyproject/galaxy/pull/12312
+.. _Pull Request 12314: https://github.com/galaxyproject/galaxy/pull/12314
+.. _Pull Request 12316: https://github.com/galaxyproject/galaxy/pull/12316
+.. _Pull Request 12317: https://github.com/galaxyproject/galaxy/pull/12317
+.. _Pull Request 12319: https://github.com/galaxyproject/galaxy/pull/12319
+.. _Pull Request 12320: https://github.com/galaxyproject/galaxy/pull/12320
+.. _Pull Request 12323: https://github.com/galaxyproject/galaxy/pull/12323
+.. _Pull Request 12324: https://github.com/galaxyproject/galaxy/pull/12324
+.. _Pull Request 12327: https://github.com/galaxyproject/galaxy/pull/12327
+.. _Pull Request 12328: https://github.com/galaxyproject/galaxy/pull/12328
+.. _Pull Request 12329: https://github.com/galaxyproject/galaxy/pull/12329
+.. _Pull Request 12333: https://github.com/galaxyproject/galaxy/pull/12333
+.. _Pull Request 12334: https://github.com/galaxyproject/galaxy/pull/12334
+.. _Pull Request 12336: https://github.com/galaxyproject/galaxy/pull/12336
+.. _Pull Request 12337: https://github.com/galaxyproject/galaxy/pull/12337
+.. _Pull Request 12338: https://github.com/galaxyproject/galaxy/pull/12338
+.. _Pull Request 12339: https://github.com/galaxyproject/galaxy/pull/12339
+.. _Pull Request 12340: https://github.com/galaxyproject/galaxy/pull/12340
+.. _Pull Request 12341: https://github.com/galaxyproject/galaxy/pull/12341
+.. _Pull Request 12344: https://github.com/galaxyproject/galaxy/pull/12344
+.. _Pull Request 12350: https://github.com/galaxyproject/galaxy/pull/12350
+.. _Pull Request 12351: https://github.com/galaxyproject/galaxy/pull/12351
+.. _Pull Request 12352: https://github.com/galaxyproject/galaxy/pull/12352
+.. _Pull Request 12354: https://github.com/galaxyproject/galaxy/pull/12354
+.. _Pull Request 12356: https://github.com/galaxyproject/galaxy/pull/12356
+.. _Pull Request 12357: https://github.com/galaxyproject/galaxy/pull/12357
+.. _Pull Request 12360: https://github.com/galaxyproject/galaxy/pull/12360
+.. _Pull Request 12361: https://github.com/galaxyproject/galaxy/pull/12361
+.. _Pull Request 12363: https://github.com/galaxyproject/galaxy/pull/12363
+.. _Pull Request 12365: https://github.com/galaxyproject/galaxy/pull/12365
+.. _Pull Request 12368: https://github.com/galaxyproject/galaxy/pull/12368
+.. _Pull Request 12370: https://github.com/galaxyproject/galaxy/pull/12370
+.. _Pull Request 12372: https://github.com/galaxyproject/galaxy/pull/12372
+.. _Pull Request 12373: https://github.com/galaxyproject/galaxy/pull/12373
+.. _Pull Request 12374: https://github.com/galaxyproject/galaxy/pull/12374
+.. _Pull Request 12376: https://github.com/galaxyproject/galaxy/pull/12376
+.. _Pull Request 12378: https://github.com/galaxyproject/galaxy/pull/12378
+.. _Pull Request 12379: https://github.com/galaxyproject/galaxy/pull/12379
+.. _Pull Request 12380: https://github.com/galaxyproject/galaxy/pull/12380
+.. _Pull Request 12381: https://github.com/galaxyproject/galaxy/pull/12381
+.. _Pull Request 12382: https://github.com/galaxyproject/galaxy/pull/12382
+.. _Pull Request 12384: https://github.com/galaxyproject/galaxy/pull/12384
+.. _Pull Request 12387: https://github.com/galaxyproject/galaxy/pull/12387
+.. _Pull Request 12388: https://github.com/galaxyproject/galaxy/pull/12388
+.. _Pull Request 12389: https://github.com/galaxyproject/galaxy/pull/12389
+.. _Pull Request 12390: https://github.com/galaxyproject/galaxy/pull/12390
+.. _Pull Request 12391: https://github.com/galaxyproject/galaxy/pull/12391
+.. _Pull Request 12392: https://github.com/galaxyproject/galaxy/pull/12392
+.. _Pull Request 12396: https://github.com/galaxyproject/galaxy/pull/12396
+.. _Pull Request 12401: https://github.com/galaxyproject/galaxy/pull/12401
+.. _Pull Request 12403: https://github.com/galaxyproject/galaxy/pull/12403
+.. _Pull Request 12405: https://github.com/galaxyproject/galaxy/pull/12405
+.. _Pull Request 12406: https://github.com/galaxyproject/galaxy/pull/12406
+.. _Pull Request 12407: https://github.com/galaxyproject/galaxy/pull/12407
+.. _Pull Request 12408: https://github.com/galaxyproject/galaxy/pull/12408
+.. _Pull Request 12409: https://github.com/galaxyproject/galaxy/pull/12409
+.. _Pull Request 12413: https://github.com/galaxyproject/galaxy/pull/12413
+.. _Pull Request 12417: https://github.com/galaxyproject/galaxy/pull/12417
+.. _Pull Request 12420: https://github.com/galaxyproject/galaxy/pull/12420
+.. _Pull Request 12422: https://github.com/galaxyproject/galaxy/pull/12422
+.. _Pull Request 12423: https://github.com/galaxyproject/galaxy/pull/12423
+.. _Pull Request 12424: https://github.com/galaxyproject/galaxy/pull/12424
+.. _Pull Request 12426: https://github.com/galaxyproject/galaxy/pull/12426
+.. _Pull Request 12427: https://github.com/galaxyproject/galaxy/pull/12427
+.. _Pull Request 12429: https://github.com/galaxyproject/galaxy/pull/12429
+.. _Pull Request 12430: https://github.com/galaxyproject/galaxy/pull/12430
+.. _Pull Request 12432: https://github.com/galaxyproject/galaxy/pull/12432
+.. _Pull Request 12433: https://github.com/galaxyproject/galaxy/pull/12433
+.. _Pull Request 12434: https://github.com/galaxyproject/galaxy/pull/12434
+.. _Pull Request 12435: https://github.com/galaxyproject/galaxy/pull/12435
+.. _Pull Request 12436: https://github.com/galaxyproject/galaxy/pull/12436
+.. _Pull Request 12438: https://github.com/galaxyproject/galaxy/pull/12438
+.. _Pull Request 12441: https://github.com/galaxyproject/galaxy/pull/12441
+.. _Pull Request 12444: https://github.com/galaxyproject/galaxy/pull/12444
+.. _Pull Request 12446: https://github.com/galaxyproject/galaxy/pull/12446
+.. _Pull Request 12447: https://github.com/galaxyproject/galaxy/pull/12447
+.. _Pull Request 12448: https://github.com/galaxyproject/galaxy/pull/12448
+.. _Pull Request 12450: https://github.com/galaxyproject/galaxy/pull/12450
+.. _Pull Request 12451: https://github.com/galaxyproject/galaxy/pull/12451
+.. _Pull Request 12456: https://github.com/galaxyproject/galaxy/pull/12456
+.. _Pull Request 12457: https://github.com/galaxyproject/galaxy/pull/12457
+.. _Pull Request 12461: https://github.com/galaxyproject/galaxy/pull/12461
+.. _Pull Request 12462: https://github.com/galaxyproject/galaxy/pull/12462
+.. _Pull Request 12463: https://github.com/galaxyproject/galaxy/pull/12463
+.. _Pull Request 12465: https://github.com/galaxyproject/galaxy/pull/12465
+.. _Pull Request 12466: https://github.com/galaxyproject/galaxy/pull/12466
+.. _Pull Request 12469: https://github.com/galaxyproject/galaxy/pull/12469
+.. _Pull Request 12471: https://github.com/galaxyproject/galaxy/pull/12471
+.. _Pull Request 12472: https://github.com/galaxyproject/galaxy/pull/12472
+.. _Pull Request 12473: https://github.com/galaxyproject/galaxy/pull/12473
+.. _Pull Request 12474: https://github.com/galaxyproject/galaxy/pull/12474
+.. _Pull Request 12481: https://github.com/galaxyproject/galaxy/pull/12481
+.. _Pull Request 12483: https://github.com/galaxyproject/galaxy/pull/12483
+.. _Pull Request 12484: https://github.com/galaxyproject/galaxy/pull/12484
+.. _Pull Request 12485: https://github.com/galaxyproject/galaxy/pull/12485
+.. _Pull Request 12488: https://github.com/galaxyproject/galaxy/pull/12488
+.. _Pull Request 12489: https://github.com/galaxyproject/galaxy/pull/12489
+.. _Pull Request 12494: https://github.com/galaxyproject/galaxy/pull/12494
+.. _Pull Request 12495: https://github.com/galaxyproject/galaxy/pull/12495
+.. _Pull Request 12496: https://github.com/galaxyproject/galaxy/pull/12496
+.. _Pull Request 12497: https://github.com/galaxyproject/galaxy/pull/12497
+.. _Pull Request 12501: https://github.com/galaxyproject/galaxy/pull/12501
+.. _Pull Request 12504: https://github.com/galaxyproject/galaxy/pull/12504
+.. _Pull Request 12505: https://github.com/galaxyproject/galaxy/pull/12505
+.. _Pull Request 12509: https://github.com/galaxyproject/galaxy/pull/12509
+.. _Pull Request 12511: https://github.com/galaxyproject/galaxy/pull/12511
+.. _Pull Request 12521: https://github.com/galaxyproject/galaxy/pull/12521
+.. _Pull Request 12532: https://github.com/galaxyproject/galaxy/pull/12532

lib/galaxy/app.py

@@ -193,10 +193,22 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication):
 
         self.sentry_client = None
         if self.config.sentry_dsn:
+            event_level = self.config.sentry_event_level.upper()
+            assert event_level in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], f"Invalid sentry event level '{self.config.sentry.event_level}'"
 
             def postfork_sentry_client():
-                import raven
-                self.sentry_client = raven.Client(self.config.sentry_dsn, transport=raven.transport.HTTPTransport)
+                import sentry_sdk
+                from sentry_sdk.integrations.logging import LoggingIntegration
+
+                sentry_logging = LoggingIntegration(
+                    level=logging.INFO,  # Capture info and above as breadcrumbs
+                    event_level=getattr(logging, event_level)  # Send errors as events
+                )
+                self.sentry_client = sentry_sdk.init(
+                    self.config.sentry_dsn,
+                    release=f"{self.config.version_major}.{self.config.version_minor}",
+                    integrations=[sentry_logging]
+                )
 
             self.application_stack.register_postfork_function(postfork_sentry_client)
 
@@ -270,9 +282,6 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
         # Load history import/export tools.
         load_lib_tools(self.toolbox)
         self.toolbox.persist_cache(register_postfork=True)
-        # If app is not job handler but uses mule messaging.
-        # Can be removed when removing mule support.
-        self.job_manager._check_jobs_at_startup()
         # visualizations registry: associates resources with visualizations, controls how to render
         self.visualizations_registry = self._register_singleton(VisualizationsRegistry, VisualizationsRegistry(
             self,
@@ -313,6 +322,13 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
                                                          self.config.oidc_config_file,
                                                          self.config.oidc_backends_config_file)
 
+        self.containers = {}
+        if self.config.enable_beta_containers_interface:
+            self.containers = build_container_interfaces(
+                self.config.containers_config_file,
+                containers_conf=self.config.containers_conf
+            )
+
         if not self.config.enable_celery_tasks and self.config.history_audit_table_prune_interval > 0:
             self.prune_history_audit_task = IntervalTask(
                 func=lambda: galaxy.model.HistoryAudit.prune(self.model.session),
@@ -324,6 +340,9 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
             self.haltables.append(("HistoryAuditTablePruneTask", self.prune_history_audit_task.shutdown))
         # Start the job manager
         self.application_stack.register_postfork_function(self.job_manager.start)
+        # If app is not job handler but uses mule messaging.
+        # Can be removed when removing mule support.
+        self.job_manager._check_jobs_at_startup()
         self.proxy_manager = ProxyManager(self.config)
 
         from galaxy.workflow import scheduling_manager
@@ -336,13 +355,6 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
         # postfork functions.
         self.application_stack.init_late_prefork()
 
-        self.containers = {}
-        if self.config.enable_beta_containers_interface:
-            self.containers = build_container_interfaces(
-                self.config.containers_config_file,
-                containers_conf=self.config.containers_conf
-            )
-
         self.interactivetool_manager = InteractiveToolManager(self)
 
         # Configure handling of signals

lib/galaxy/config/__init__.py

@@ -45,10 +45,7 @@ from galaxy.util.properties import (
     running_from_source,
 )
 from galaxy.web.formatting import expand_pretty_datetime_format
-from galaxy.web_stack import (
-    get_stack_facts,
-    register_postfork_function
-)
+from galaxy.web_stack import get_stack_facts
 from ..version import VERSION_MAJOR, VERSION_MINOR
 
 log = logging.getLogger(__name__)
@@ -1106,7 +1103,6 @@ def configure_logging(config):
     """
     # Get root logger
     logging.addLevelName(LOGLV_TRACE, "TRACE")
-    root = logging.getLogger()
     # PasteScript will have already configured the logger if the
     # 'loggers' section was found in the config file, otherwise we do
     # some simple setup using the 'log_*' values from the config.
@@ -1129,11 +1125,6 @@ def configure_logging(config):
                 conf['filename'] = conf.pop('filename_template').format(**get_stack_facts(config=config))
                 logging_conf['handlers'][name] = conf
         logging.config.dictConfig(logging_conf)
-    if getattr(config, "sentry_dsn", None):
-        from raven.handlers.logging import SentryHandler
-        sentry_handler = SentryHandler(config.sentry_dsn)
-        sentry_handler.setLevel(logging.WARN)
-        register_postfork_function(root.addHandler, sentry_handler)
 
 
 class ConfiguresGalaxyMixin:

lib/galaxy/config/config_manage.py

@@ -293,8 +293,8 @@ OPTION_ACTIONS = {
     'enable_beta_job_managers': _DeprecatedAndDroppedAction(),
     'enable_legacy_sample_tracking_api': _DeprecatedAction(),
     'enable_new_user_preferences': _DeprecatedAndDroppedAction(),
-    'force_beta_workflow_scheduled_for_collections': _DeprecatedAction(),
-    'force_beta_workflow_scheduled_min_steps': _DeprecatedAction(),
+    'force_beta_workflow_scheduled_for_collections': _DeprecatedAndDroppedAction(),
+    'force_beta_workflow_scheduled_min_steps': _DeprecatedAndDroppedAction(),
     'history_local_serial_workflow_scheduling': _ProductionPerformance(),
     'allow_library_path_paste': _RenameAction("allow_path_paste"),
     'trust_ipython_notebook_conversion': _RenameAction("trust_jupyter_notebook_conversion"),

lib/galaxy/config/sample/error_report.yml.sample

@@ -15,7 +15,7 @@
 # verbose/user_submission, but those are not necessary to provide.
 
 # The default Email bug reporter. By default, the standard
-# configuration is taken from your galaxy.ini
+# configuration is taken from your galaxy.yml
 - type: email
   verbose: true
   user_submission: true
@@ -29,10 +29,9 @@
 #   directory: /tmp/reports/
 
 # Submit error reports to sentry. If a sentry_dsn is configured in your
-# galaxy.ini, then Galaxy will submit the job error to Sentry. You may supply a
-# separate DSN for tool reports by supplying a ``custom_dsn`` parameter.
-- type: sentry
-  user_submission: false
+# galaxy.yml, then Galaxy will submit the job error to Sentry.
+# - type: sentry
+#   user_submission: false
 
 # InfluxDB error reporting backend. You will need to `pip install
 # influxdb` in the galaxy virtualenv yourself. This sends well tagged

lib/galaxy/config/sample/galaxy.yml.sample

@@ -1388,12 +1388,9 @@ galaxy:
   # <project_name> -> Settings -> API Keys.
   #sentry_dsn: null
 
-  # Sentry slow request logging.  Requests slower than the threshold
-  # indicated below will be sent as events to the configured Sentry
-  # server (above, sentry_dsn).  A value of '0' is disabled.  For
-  # example, you would set this to .005 to log all queries taking longer
-  # than 5 milliseconds.
-  #sentry_sloreq_threshold: 0.0
+  # Determines the minimum log level that will be sent as an event to
+  # Sentry. Possible values are DEBUG, INFO, WARNING, ERROR or CRITICAL.
+  #sentry_event_level: ERROR
 
   # Log to statsd Statsd is an external statistics aggregator
   # (https://github.com/etsy/statsd) Enabling the following options will

lib/galaxy/datatypes/data.py

@@ -46,6 +46,10 @@ DOWNLOAD_FILENAME_PATTERN_COLLECTION_ELEMENT = "Galaxy${hdca_hid}-[${hdca_name}_
 DEFAULT_MAX_PEEK_SIZE = 1000000  # 1 MB
 
 
+class DatatypeConverterNotFoundException(Exception):
+    pass
+
+
 class DatatypeValidation:
 
     def __init__(self, state, message):
@@ -638,7 +642,7 @@ class Data(metaclass=DataMeta):
         converter = trans.app.datatypes_registry.get_converter_by_target_type(original_dataset.ext, target_type)
 
         if converter is None:
-            raise Exception(f"A converter does not exist for {original_dataset.ext} to {target_type}.")
+            raise DatatypeConverterNotFoundException(f"A converter does not exist for {original_dataset.ext} to {target_type}.")
 
         params, input_name = get_params_and_input_name(converter, deps, target_context)
 

lib/galaxy/datatypes/tabular.py

@@ -1362,7 +1362,13 @@ class CMAP(TabularData):
     file_ext = "cmap"
 
     def sniff_prefix(self, file_prefix):
-        return file_prefix.startswith('# CMAP File Version:')
+        handle = file_prefix.string_io()
+        for line in handle:
+            if not line.startswith('#'):
+                return False
+            if line.startswith('# CMAP File Version:'):
+                return True
+        return False
 
     def set_meta(self, dataset, overwrite=True, skip=None, max_data_lines=7, **kwd):
         if dataset.has_data():

lib/galaxy/dependencies/__init__.py

@@ -191,7 +191,7 @@ class ConditionalDependencies:
     def check_fluent_logger(self):
         return asbool(self.config["fluent_log"])
 
-    def check_raven(self):
+    def check_sentry_sdk(self):
         return self.config.get("sentry_dsn", None) is not None
 
     def check_statsd(self):

lib/galaxy/dependencies/conditional-requirements.txt

@@ -1,8 +1,8 @@
 # These dependencies are only required when certain config options are set
-psycopg2-binary==2.8.4
+psycopg2-binary==2.9.1
 mysqlclient
 fluent-logger
-raven
+sentry-sdk
 pbs_python
 drmaa
 statsd

lib/galaxy/dependencies/dev-requirements.txt

@@ -174,6 +174,7 @@ pytest-html==3.1.1; python_version >= "3.6"
 pytest-json-report==1.2.4
 pytest-metadata==1.11.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 pytest-mock==3.6.0; python_version >= "3.6"
+python-multipart==0.0.5
 pytest-postgresql==2.6.1; python_version >= "3.6"
 pytest-pythonpath==0.7.3
 pytest-shard==0.1.2; python_version >= "3.6"
@@ -226,7 +227,7 @@ sphinxcontrib-jsmath==1.0.1; python_version >= "3.5"
 sphinxcontrib-qthelp==1.0.3; python_version >= "3.5"
 sphinxcontrib-serializinghtml==1.1.4; python_version >= "3.5"
 sqlalchemy-migrate==0.13.0
-sqlalchemy==1.4.20
+sqlalchemy==1.4.25
 sqlitedict==1.7.0
 sqlparse==0.4.2; python_version >= "3.5"
 starlette-context==0.3.2; python_version >= "3.7"

lib/galaxy/dependencies/pinned-requirements.txt

@@ -152,6 +152,7 @@ pysam==0.16.0.1
 python-dateutil==2.8.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.3.0"
 python-jose==3.2.0
 python-keystoneclient==4.1.1; python_version >= "3.6"
+python-multipart==0.0.5
 python-neutronclient==7.2.1; python_version >= "3.6"
 python-novaclient==17.2.1; python_version >= "3.6"
 python-swiftclient==3.10.1
@@ -182,7 +183,7 @@ six==1.15.0; python_version >= "3.6" and python_full_version < "3.0.0" and pytho
 social-auth-core==3.3.0
 sortedcontainers==2.3.0
 sqlalchemy-migrate==0.13.0
-sqlalchemy==1.4.20
+sqlalchemy==1.4.25
 sqlitedict==1.7.0
 sqlparse==0.4.2; python_version >= "3.5"
 starlette-context==0.3.2; python_version >= "3.7"

lib/galaxy/files/__init__.py

@@ -146,6 +146,8 @@ class ConfiguredFileSources:
     def plugins_to_dict(self, for_serialization=False, user_context=None):
         rval = []
         for file_source in self._file_sources:
+            if not file_source.user_has_access(user_context):
+                continue
             el = file_source.to_dict(for_serialization=for_serialization, user_context=user_context)
             rval.append(el)
         return rval

lib/galaxy/files/sources/__init__.py

@@ -71,6 +71,8 @@ class BaseFilesSource(FilesSource):
         return self.writable
 
     def user_has_access(self, user_context) -> bool:
+        if user_context is None and self.user_context_required:
+            return False
         return (
             user_context is None
             or user_context.is_admin
@@ -80,6 +82,10 @@ class BaseFilesSource(FilesSource):
             )
         )
 
+    @property
+    def user_context_required(self) -> bool:
+        return self.requires_roles is not None or self.requires_groups is not None
+
     def get_uri_root(self):
         prefix = self.get_prefix()
         scheme = self.get_scheme()
@@ -164,7 +170,11 @@ class BaseFilesSource(FilesSource):
         pass
 
     def _check_user_access(self, user_context):
-        """Raises an exception if the given user doesn't have the rights to access this file source."""
+        """Raises an exception if the given user doesn't have the rights to access this file source.
+
+        Warning: if the user_context is None, then the check is skipped. This is due to tool executions context
+        not having access to the user_context. The validation will be done when checking the tool parameters.
+        """
         if user_context is not None and not self.user_has_access(user_context):
             raise ItemAccessibilityException(f"User {user_context.username} has no access to file source.")
 

lib/galaxy/jobs/__init__.py

@@ -1142,10 +1142,10 @@ class JobWrapper(HasResourceParameters):
         job = self._load_job()
 
         def get_special():
-            special = self.sa_session.query(model.JobExportHistoryArchive).filter_by(job=job).first()
-            if not special:
-                special = self.sa_session.query(model.GenomeIndexToolData).filter_by(job=job).first()
-            return special
+            jeha = self.sa_session.query(model.JobExportHistoryArchive).filter_by(job=job).first()
+            if jeha:
+                return jeha.fda
+            return self.sa_session.query(model.GenomeIndexToolData).filter_by(job=job).first()
 
         tool_evaluator = self._get_tool_evaluator(job)
         compute_environment = compute_environment or self.default_compute_environment(job)
@@ -1752,7 +1752,7 @@ class JobWrapper(HasResourceParameters):
         # the state instead.
 
         for pja in job.post_job_actions:
-            ActionBox.execute(self.app, self.sa_session, pja.post_job_action, job)
+            ActionBox.execute(self.app, self.sa_session, pja.post_job_action, job, final_job_state=final_job_state)
         # Flush all the dataset and job changes above.  Dataset state changes
         # will now be seen by the user.
         self.sa_session.flush()

lib/galaxy/jobs/actions/post.py

@@ -24,7 +24,7 @@ class DefaultJobAction:
     verbose_name = "Default Job"
 
     @classmethod
-    def execute(cls, app, sa_session, action, job, replacement_dict=None):
+    def execute(cls, app, sa_session, action, job, replacement_dict=None, final_job_state=None):
         pass
 
     @classmethod
@@ -43,7 +43,7 @@ class EmailAction(DefaultJobAction):
     verbose_name = "Email Notification"
 
     @classmethod
-    def execute(cls, app, sa_session, action, job, replacement_dict):
+    def execute(cls, app, sa_session, action, job, replacement_dict, final_job_state=None):
         try:
             frm = app.config.email_from
             history_id_encoded = app.security.encode_id(job.history_id)
@@ -78,7 +78,7 @@ class ValidateOutputsAction(DefaultJobAction):
     verbose_name = "Validate Tool Outputs"
 
     @classmethod
-    def execute(cls, app, sa_session, action, job, replacement_dict):
+    def execute(cls, app, sa_session, action, job, replacement_dict, final_job_state=None):
         # no-op: needs to inject metadata handling parameters ahead of time.
         pass
 
@@ -92,7 +92,7 @@ class ChangeDatatypeAction(DefaultJobAction):
     verbose_name = "Change Datatype"
 
     @classmethod
-    def execute(cls, app, sa_session, action, job, replacement_dict):
+    def execute(cls, app, sa_session, action, job, replacement_dict, final_job_state=None):
         for dataset_assoc in job.output_datasets:
             if action.output_name == '' or dataset_assoc.name == action.output_name:
                 app.datatypes_registry.change_datatype(dataset_assoc.dataset, action.action_arguments['newtype'])
@@ -113,7 +113,7 @@ class RenameDatasetAction(DefaultJobAction):
     verbose_name = "Rename Dataset"
 
     @classmethod
-    def execute_on_mapped_over(cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict):
+    def execute_on_mapped_over(cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict, final_job_state=None):
         # Prevent renaming a dataset to the empty string.
         input_names = {}
         #  Lookp through inputs find one with "to_be_replaced" input
@@ -207,7 +207,7 @@ class RenameDatasetAction(DefaultJobAction):
         return new_name
 
     @classmethod
-    def execute(cls, app, sa_session, action, job, replacement_dict):
+    def execute(cls, app, sa_session, action, job, replacement_dict, final_job_state=None):
         input_names = {}
         #  Lookp through inputs find one with "to_be_replaced" input
         #  variable name, and get the replacement name
@@ -247,17 +247,14 @@ class HideDatasetAction(DefaultJobAction):
     verbose_name = "Hide Dataset"
 
     @classmethod
-    def execute(cls, app, sa_session, action, job, replacement_dict):
-        for dataset_assoc in job.output_datasets:
-            if dataset_assoc.dataset.state != dataset_assoc.dataset.states.ERROR and (action.output_name == '' or dataset_assoc.name == action.output_name):
-                dataset_assoc.dataset.visible = False
-
-        for dataset_collection_assoc in job.output_dataset_collection_instances:
-            if action.output_name == '' or dataset_collection_assoc.name == action.output_name:
-                dataset_collection_assoc.dataset_collection_instance.visible = False
+    def execute(cls, app, sa_session, action, job, replacement_dict, final_job_state=None):
+        if final_job_state != job.states.ERROR:
+            for output_association in job.output_datasets + job.output_dataset_collection_instances:
+                if action.output_name == '' or output_association.name == action.output_name:
+                    output_association.item.visible = False
 
     @classmethod
-    def execute_on_mapped_over(cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict):
+    def execute_on_mapped_over(cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict, final_job_state=None):
         for name, step_output in step_outputs.items():
             if action.output_name == '' or name == action.output_name:
                 step_output.visible = False
@@ -273,17 +270,13 @@ class DeleteDatasetAction(DefaultJobAction):
     verbose_name = "Delete Dataset"
 
     @classmethod
-    def execute(cls, app, sa_session, action, job, replacement_dict):
-        for dataset_assoc in job.output_datasets:
-            if action.output_name == '' or dataset_assoc.name == action.output_name:
-                dataset_assoc.dataset.deleted = True
-
-        for dataset_collection_assoc in job.output_dataset_collection_instances:
-            if action.output_name == '' or dataset_collection_assoc.name == action.output_name:
-                dataset_collection_assoc.dataset_collection_instance.deleted = True
+    def execute(cls, app, sa_session, action, job, replacement_dict, final_job_state=None):
+        for output_association in job.output_datasets + job.output_dataset_collection_instances:
+            if action.output_name == '' or output_association.name == action.output_name:
+                output_association.item.deleted = True
 
     @classmethod
-    def execute_on_mapped_over(cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict):
+    def execute_on_mapped_over(cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict, final_job_state=None):
         for name, step_output in step_outputs.items():
             if action.output_name == '' or name == action.output_name:
                 step_output.deleted = True
@@ -298,7 +291,7 @@ class ColumnSetAction(DefaultJobAction):
     verbose_name = "Assign Columns"
 
     @classmethod
-    def execute(cls, app, sa_session, action, job, replacement_dict):
+    def execute(cls, app, sa_session, action, job, replacement_dict, final_job_state=None):
         for dataset_assoc in job.output_datasets:
             if action.output_name == '' or dataset_assoc.name == action.output_name:
                 for k, v in action.action_arguments.items():
@@ -321,7 +314,7 @@ class SetMetadataAction(DefaultJobAction):
     # DBTODO Setting of Metadata is currently broken and disabled.  It should not be used (yet).
 
     @classmethod
-    def execute(cls, app, sa_session, action, job, replacement_dict):
+    def execute(cls, app, sa_session, action, job, replacement_dict, final_job_state=None):
         for data in job.output_datasets:
             data.set_metadata(action.action_arguments['newtype'])
 
@@ -331,7 +324,7 @@ class DeleteIntermediatesAction(DefaultJobAction):
     verbose_name = "Delete Non-Output Completed Intermediate Steps"
 
     @classmethod
-    def execute(cls, app, sa_session, action, job, replacement_dict):
+    def execute(cls, app, sa_session, action, job, replacement_dict, final_job_state=None):
         # TODO Optimize this later.  Just making it work for now.
         # TODO Support purging as well as deletion if user_purge is enabled.
         # Dataset candidates for deletion must be
@@ -405,7 +398,7 @@ class TagDatasetAction(DefaultJobAction):
     direction = "to"
 
     @classmethod
-    def execute_on_mapped_over(cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict):
+    def execute_on_mapped_over(cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict, final_job_state=None):
         if action.action_arguments:
             tags = [t.replace('#', 'name:') if t.startswith('#') else t for t in [t.strip() for t in action.action_arguments.get('tags', '').split(',') if t.strip()]]
             if tags:
@@ -414,7 +407,7 @@ class TagDatasetAction(DefaultJobAction):
                         cls._execute(trans.app, trans.user, step_output, tags)
 
     @classmethod
-    def execute(cls, app, sa_session, action, job, replacement_dict):
+    def execute(cls, app, sa_session, action, job, replacement_dict, final_job_state=None):
         if action.action_arguments:
             tags = [t.replace('#', 'name:') if t.startswith('#') else t for t in [t.strip() for t in action.action_arguments.get('tags', '').split(',') if t.strip()]]
             if tags:
@@ -507,11 +500,11 @@ class ActionBox:
         return npd
 
     @classmethod
-    def execute_on_mapped_over(cls, trans, sa_session, pja, step_inputs, step_outputs, replacement_dict=None):
+    def execute_on_mapped_over(cls, trans, sa_session, pja, step_inputs, step_outputs, replacement_dict=None, final_job_state=None):
         if pja.action_type in ActionBox.actions:
-            ActionBox.actions[pja.action_type].execute_on_mapped_over(trans, sa_session, pja, step_inputs, step_outputs, replacement_dict)
+            ActionBox.actions[pja.action_type].execute_on_mapped_over(trans, sa_session, pja, step_inputs, step_outputs, replacement_dict, final_job_state=final_job_state)
 
     @classmethod
-    def execute(cls, app, sa_session, pja, job, replacement_dict=None):
+    def execute(cls, app, sa_session, pja, job, replacement_dict=None, final_job_state=None):
         if pja.action_type in ActionBox.actions:
-            ActionBox.actions[pja.action_type].execute(app, sa_session, pja, job, replacement_dict)
+            ActionBox.actions[pja.action_type].execute(app, sa_session, pja, job, replacement_dict, final_job_state=final_job_state)

lib/galaxy/jobs/handler.py

@@ -64,6 +64,7 @@ class JobHandler(JobHandlerI):
         self.job_stop_queue = JobHandlerStopQueue(app, self.dispatcher)
 
     def start(self):
+        self.dispatcher.start()
         self.job_queue.start()
         self.job_stop_queue.start()
 
@@ -1025,6 +1026,10 @@ class DefaultJobDispatcher:
         self.app.job_config.convert_legacy_destinations(self.job_runners)
         log.debug(f"Loaded job runners plugins: {':'.join(self.job_runners.keys())}")
 
+    def start(self):
+        for runner in self.job_runners.values():
+            runner.start()
+
     def __get_runner_name(self, job_wrapper):
         if job_wrapper.can_split():
             runner_name = "tasks"

lib/galaxy/jobs/manager.py

@@ -24,11 +24,7 @@ class JobManager:
     def __init__(self, app: MinimalManagerApp):
         self.app = app
         self.job_lock = False
-        if self.app.is_job_handler:
-            log.debug("Initializing job handler")
-            self.job_handler = handler.JobHandler(app)
-        else:
-            self.job_handler = NoopHandler()
+        self.job_handler = NoopHandler()
 
     def _check_jobs_at_startup(self):
         if not self.app.is_job_handler:
@@ -47,7 +43,10 @@ class JobManager:
                 self.enqueue(job, tool)
 
     def start(self):
-        self.job_handler.start()
+        if self.app.is_job_handler:
+            log.debug("Initializing job handler")
+            self.job_handler = handler.JobHandler(self.app)
+            self.job_handler.start()
 
     def _queue_callback(self, job, tool_id):
         self.job_handler.job_queue.put(job.id, tool_id)

lib/galaxy/jobs/runners/__init__.py

@@ -67,6 +67,8 @@ class RunnerParams(ParamsWithSpecs):
 
 
 class BaseJobRunner:
+
+    start_methods = ['_init_monitor_thread', '_init_worker_threads']
     DEFAULT_SPECS = dict(recheck_missing_job_retries=dict(map=int, valid=lambda x: int(x) >= 0, default=0))
 
     def __init__(self, app, nworkers, **kwargs):
@@ -84,6 +86,10 @@ class BaseJobRunner:
         self.runner_params = RunnerParams(specs=runner_param_specs, params=kwargs)
         self.runner_state_handlers = build_state_handlers()
 
+    def start(self):
+        for start_method in self.start_methods:
+            getattr(self, start_method, lambda: None)()
+
     def _init_worker_threads(self):
         """Start ``nworkers`` worker threads.
         """
@@ -93,7 +99,7 @@ class BaseJobRunner:
         for i in range(self.nworkers):
             worker = threading.Thread(name="%s.work_thread-%d" % (self.runner_name, i), target=self.run_next)
             worker.daemon = True
-            self.app.application_stack.register_postfork_function(worker.start)
+            worker.start()
             self.work_threads.append(worker)
 
     def _alive_worker_threads(self, cycle=False):

lib/galaxy/jobs/runners/chronos.py

@@ -142,8 +142,6 @@ class ChronosJobRunner(AsynchronousJobRunner):
             username=self.runner_params.get('username'),
             password=self.runner_params.get('password'),
             proto=protocol)
-        self._init_monitor_thread()
-        self._init_worker_threads()
 
     @handle_exception_call
     def queue_job(self, job_wrapper):

lib/galaxy/jobs/runners/cli.py

@@ -33,8 +33,6 @@ class ShellJobRunner(AsynchronousJobRunner):
         super().__init__(app, nworkers)
 
         self.cli_interface = CliInterface()
-        self._init_monitor_thread()
-        self._init_worker_threads()
 
     def get_cli_plugins(self, shell_params, job_params):
         return self.cli_interface.get_plugins(shell_params, job_params)

lib/galaxy/jobs/runners/condor.py

@@ -51,12 +51,6 @@ class CondorJobRunner(AsynchronousJobRunner):
     """
     runner_name = "CondorRunner"
 
-    def __init__(self, app, nworkers):
-        """Initialize this job runner and start the monitor thread"""
-        super().__init__(app, nworkers)
-        self._init_monitor_thread()
-        self._init_worker_threads()
-
     def queue_job(self, job_wrapper):
         """Create job script and submit it to the DRM"""
 

lib/galaxy/jobs/runners/drmaa.py

@@ -95,8 +95,6 @@ class DRMAAJobRunner(AsynchronousJobRunner):
 
         self.userid = None
 
-        self._init_monitor_thread()
-        self._init_worker_threads()
         self.redact_email_in_job_name = self.app.config.redact_email_in_job_name
 
     def url_to_destination(self, url):

lib/galaxy/jobs/runners/godocker.py

@@ -138,16 +138,6 @@ class GodockerJobRunner(AsynchronousJobRunner):
         # godocker API login call
         self.auth = self.login(self.runner_params["key"], self.runner_params["user"], self.runner_params["godocker_master"])
 
-        if not self.auth:
-            log.error("Authentication failure, GoDocker runner cannot be started")
-        else:
-            """ Following methods starts threads.
-                These methods invoke threading.Thread(name,target)
-                      which in turn invokes methods monitor() and run_next().
-            """
-            self._init_monitor_thread()
-            self._init_worker_threads()
-
     def queue_job(self, job_wrapper):
         """ Create job script and submit it to godocker """
         if not self.prepare_job(job_wrapper, include_metadata=False, include_work_dir_outputs=True, modify_command_for_container=False):
@@ -324,7 +314,7 @@ class GodockerJobRunner(AsynchronousJobRunner):
         g_auth = Godocker(server, login, apikey, noCert)
         auth = g_auth.http_post_request("/api/1.0/authenticate", data, {'Content-type': 'application/json', 'Accept': 'application/json'})
         if not auth:
-            log.error("GoDocker authentication Error.")
+            raise Exception("Authentication failure, GoDocker runner cannot be started")
         else:
             log.debug("GoDocker authentication successful.")
             token = auth.json()['token']

lib/galaxy/jobs/runners/kubernetes.py

@@ -103,8 +103,6 @@ class KubernetesJobRunner(AsynchronousJobRunner):
         self._fs_group = self.__get_fs_group()
         self._default_pull_policy = self.__get_pull_policy()
 
-        self._init_monitor_thread()
-        self._init_worker_threads()
         self.setup_volumes()
 
     def setup_volumes(self):

lib/galaxy/jobs/runners/local.py

@@ -52,7 +52,6 @@ class LocalJobRunner(BaseJobRunner):
             self._environ['TEMP'] = os.path.abspath(tempfile.gettempdir())
 
         super().__init__(app, nworkers)
-        self._init_worker_threads()
 
     def __command_line(self, job_wrapper):
         """

lib/galaxy/jobs/runners/pbs.py

@@ -102,8 +102,6 @@ class PBSJobRunner(AsynchronousJobRunner):
 
         # Proceed with general initialization
         super().__init__(app, nworkers)
-        self._init_monitor_thread()
-        self._init_worker_threads()
 
     @property
     def default_pbs_server(self):

lib/galaxy/jobs/runners/pulsar.py

@@ -188,6 +188,7 @@ PARAMETER_SPECIFICATION_IGNORED = object()
 class PulsarJobRunner(AsynchronousJobRunner):
     """Base class for pulsar job runners."""
 
+    start_methods = ['_init_worker_threads', '_init_client_manager', '_monitor']
     runner_name = "PulsarJobRunner"
     default_build_pulsar_app = False
     use_mq = False
@@ -196,15 +197,12 @@ class PulsarJobRunner(AsynchronousJobRunner):
     def __init__(self, app, nworkers, **kwds):
         """Start the job runner."""
         super().__init__(app, nworkers, runner_param_specs=PULSAR_PARAM_SPECS, **kwds)
-        self._init_worker_threads()
         galaxy_url = self.runner_params.galaxy_url
         if not galaxy_url:
             galaxy_url = app.config.galaxy_infrastructure_url
         if galaxy_url:
             galaxy_url = galaxy_url.rstrip("/")
         self.galaxy_url = galaxy_url
-        self.__init_client_manager()
-        self._monitor()
 
     def _monitor(self):
         if self.use_mq:
@@ -218,7 +216,7 @@ class PulsarJobRunner(AsynchronousJobRunner):
         else:
             self._init_noop_monitor()
 
-    def __init_client_manager(self):
+    def _init_client_manager(self):
         pulsar_conf = self.runner_params.get('pulsar_app_config', None)
         pulsar_conf_file = None
         if pulsar_conf is None:

lib/galaxy/managers/base.py

@@ -46,7 +46,6 @@ from galaxy import exceptions
 from galaxy import model
 from galaxy.model import tool_shed_install
 from galaxy.schema import FilterQueryParams
-from galaxy.schema.fields import EncodedDatabaseIdField
 from galaxy.security.idencoding import IdEncodingHelper
 from galaxy.structured_app import BasicApp, MinimalManagerApp
 from galaxy.util import namedtuple
@@ -1204,41 +1203,3 @@ class SortableManager:
         """Return an ORM compatible order_by clause using the given string (i.e.: 'name-dsc,create_time').
         This must be implemented by the manager."""
         raise NotImplementedError
-
-
-class ServiceBase:
-    """Base class with common logic and utils reused by other Services."""
-
-    def __init__(self, security: IdEncodingHelper):
-        self.security = security
-
-    def decode_id(self, id: EncodedDatabaseIdField) -> int:
-        """Decodes a previously encoded database ID."""
-        return decode_with_security(self.security, id)
-
-    def encode_id(self, id: int) -> EncodedDatabaseIdField:
-        """Encodes a raw database ID."""
-        return encode_with_security(self.security, id)
-
-    def decode_ids(self, ids: List[EncodedDatabaseIdField]) -> List[int]:
-        """
-        Decodes all encoded IDs in the given list.
-        """
-        return [self.decode_id(id) for id in ids]
-
-    def encode_all_ids(self, rval, recursive: bool = False):
-        """
-        Encodes all integer values in the dict rval whose keys are 'id' or end with '_id'
-
-        It might be useful to turn this in to a decorator
-        """
-        return self.security.encode_all_ids(rval, recursive=recursive)
-
-    def build_order_by(self, manager: SortableManager, order_by_query: Optional[str] = None):
-        """Returns an ORM compatible order_by clause using the order attribute and the given manager.
-
-        The manager has to implement the `parse_order_by` function to support all the sortable model attributes."""
-        ORDER_BY_SEP_CHAR = ','
-        if order_by_query and ORDER_BY_SEP_CHAR in order_by_query:
-            return [manager.parse_order_by(o) for o in order_by_query.split(ORDER_BY_SEP_CHAR)]
-        return manager.parse_order_by(order_by_query)

lib/galaxy/managers/configuration.py

@@ -18,8 +18,8 @@ from typing import (
 from galaxy.app import MinimalManagerApp
 from galaxy.managers import base
 from galaxy.managers.context import ProvidesUserContext
+from galaxy.schema import SerializationParams
 from galaxy.schema.fields import EncodedDatabaseIdField
-from galaxy.schema.types import SerializationParams
 from galaxy.web.framework.base import server_starttime
 
 log = logging.getLogger(__name__)
@@ -42,7 +42,7 @@ class ConfigurationManager:
         host = getattr(trans, "host", None)
         serializer_class = AdminConfigSerializer if is_admin else ConfigSerializer
         serializer = serializer_class(self._app)
-        return serializer.serialize_to_view(self._app.config, host=host, **serialization_params)
+        return serializer.serialize_to_view(self._app.config, host=host, **serialization_params.dict())
 
     def version(self) -> Dict[str, Any]:
         version_info = {

lib/galaxy/managers/folders.py

@@ -2,10 +2,6 @@
 Manager and Serializer for Library Folders.
 """
 import logging
-from typing import (
-    Optional,
-    Union,
-)
 
 from sqlalchemy.orm.exc import (
     MultipleResultsFound,
@@ -21,19 +17,7 @@ from galaxy.exceptions import (
     ItemAccessibilityException,
     MalformedId,
     RequestParameterInvalidException,
-    RequestParameterMissingException,
 )
-from galaxy.managers.roles import RoleManager
-from galaxy.schema.fields import EncodedDatabaseIdField
-from galaxy.schema.schema import (
-    CreateLibraryFolderPayload,
-    LibraryAvailablePermissions,
-    LibraryFolderCurrentPermissions,
-    LibraryFolderDetails,
-    LibraryPermissionScope,
-    UpdateLibraryFolderPayload,
-)
-from galaxy.structured_app import StructuredApp
 
 log = logging.getLogger(__name__)
 
@@ -321,251 +305,3 @@ class FolderManager:
         :rtype:    int
         """
         return self.decode_folder_id(trans, self.cut_the_prefix(encoded_folder_id))
-
-
-class FoldersService:
-    """Common interface/service logic for interactions with library folders in the context of the API.
-    Provides the logic of the actions invoked by API controllers and uses type definitions
-    and pydantic models to declare its parameters and return types.
-    """
-
-    def __init__(self, app: StructuredApp, folder_manager: FolderManager, role_manager: RoleManager) -> None:
-        self._app = app
-        self.folder_manager = folder_manager
-        self.role_manager = role_manager
-
-    def show(self, trans, id: EncodedDatabaseIdField) -> LibraryFolderDetails:
-        """
-        Displays information about a folder.
-
-        :param  id:      the folder's encoded id (required)
-        :type   id:      an encoded id string (has to be prefixed by 'F')
-
-        :returns:   dictionary including details of the folder
-        :rtype:     dict
-        """
-        folder_id = self.folder_manager.cut_and_decode(trans, id)
-        folder = self.folder_manager.get(trans, folder_id, check_manageable=False, check_accessible=True)
-        return_dict = self.folder_manager.get_folder_dict(trans, folder)
-        return LibraryFolderDetails.parse_obj(return_dict)
-
-    def create(
-        self,
-        trans,
-        encoded_parent_folder_id: EncodedDatabaseIdField,
-        payload: CreateLibraryFolderPayload
-    ) -> LibraryFolderDetails:
-        """
-        Create a new folder object underneath the one specified in the parameters.
-
-        :param  encoded_parent_folder_id:      (required) the parent folder's id
-        :type   encoded_parent_folder_id:      an encoded id string (should be prefixed by 'F')
-        :param   payload: dictionary structure containing:
-
-            :param  name:                          (required) the name of the new folder
-            :type   name:                          str
-            :param  description:                   the description of the new folder
-            :type   description:                   str
-
-        :type       dictionary
-        :returns:   information about newly created folder, notably including ID
-        :rtype:     dictionary
-        :raises: RequestParameterMissingException
-        """
-        decoded_parent_folder_id = self.folder_manager.cut_and_decode(trans, encoded_parent_folder_id)
-        parent_folder = self.folder_manager.get(trans, decoded_parent_folder_id)
-        new_folder = self.folder_manager.create(trans, parent_folder.id, payload.name, payload.description)
-        return_dict = self.folder_manager.get_folder_dict(trans, new_folder)
-        return LibraryFolderDetails.parse_obj(return_dict)
-
-    def get_permissions(
-        self,
-        trans,
-        encoded_folder_id: EncodedDatabaseIdField,
-        scope: Optional[LibraryPermissionScope] = LibraryPermissionScope.current,
-        page: Optional[int] = 1,
-        page_limit: Optional[int] = 10,
-        query: Optional[str] = None,
-    ) -> Union[LibraryFolderCurrentPermissions, LibraryAvailablePermissions]:
-        """
-        Load all permissions for the given folder id and return it.
-
-        :param  encoded_folder_id:     the encoded id of the folder
-        :type   encoded_folder_id:     an encoded id string
-
-        :param  scope:      either 'current' or 'available'
-        :type   scope:      string
-
-        :returns:   dictionary with all applicable permissions' values
-        :rtype:     dictionary
-
-        :raises: InsufficientPermissionsException
-        """
-        current_user_roles = trans.get_current_user_roles()
-        is_admin = trans.user_is_admin
-        decoded_folder_id = self.folder_manager.cut_and_decode(trans, encoded_folder_id)
-        folder = self.folder_manager.get(trans, decoded_folder_id)
-
-        if not (is_admin or trans.app.security_agent.can_manage_library_item(current_user_roles, folder)):
-            raise InsufficientPermissionsException('You do not have proper permission to access permissions of this folder.')
-
-        if scope is None or scope == LibraryPermissionScope.current:
-            current_permissions = self.folder_manager.get_current_roles(trans, folder)
-            return LibraryFolderCurrentPermissions.parse_obj(current_permissions)
-        #  Return roles that are available to select.
-        elif scope == LibraryPermissionScope.available:
-            roles, total_roles = trans.app.security_agent.get_valid_roles(trans, folder, query, page, page_limit)
-            return_roles = []
-            for role in roles:
-                role_id = trans.security.encode_id(role.id)
-                return_roles.append(dict(id=role_id, name=role.name, type=role.type))
-            return LibraryAvailablePermissions(roles=return_roles, page=page, page_limit=page_limit, total=total_roles)
-        else:
-            raise RequestParameterInvalidException("The value of 'scope' parameter is invalid. Allowed values: current, available")
-
-    def set_permissions(
-        self,
-        trans,
-        encoded_folder_id: EncodedDatabaseIdField,
-        payload: dict
-    ) -> LibraryFolderCurrentPermissions:
-        """
-        Set permissions of the given folder to the given role ids.
-
-        :param  encoded_folder_id:      the encoded id of the folder to set the permissions of
-        :type   encoded_folder_id:      an encoded id string
-        :param   payload: dictionary structure containing:
-
-            :param  action:            (required) describes what action should be performed
-            :type   action:            string
-            :param  add_ids[]:         list of Role.id defining roles that should have add item permission on the folder
-            :type   add_ids[]:         string or list
-            :param  manage_ids[]:      list of Role.id defining roles that should have manage permission on the folder
-            :type   manage_ids[]:      string or list
-            :param  modify_ids[]:      list of Role.id defining roles that should have modify permission on the folder
-            :type   modify_ids[]:      string or list
-
-        :type       dictionary
-        :returns:   dict of current roles for all available permission types.
-        :rtype:     dictionary
-        :raises: RequestParameterInvalidException, InsufficientPermissionsException, RequestParameterMissingException
-        """
-
-        is_admin = trans.user_is_admin
-        current_user_roles = trans.get_current_user_roles()
-        decoded_folder_id = self.folder_manager.cut_and_decode(trans, encoded_folder_id)
-        folder = self.folder_manager.get(trans, decoded_folder_id)
-        if not (is_admin or trans.app.security_agent.can_manage_library_item(current_user_roles, folder)):
-            raise InsufficientPermissionsException('You do not have proper permission to modify permissions of this folder.')
-
-        new_add_roles_ids = util.listify(payload.get('add_ids[]', None))
-        new_manage_roles_ids = util.listify(payload.get('manage_ids[]', None))
-        new_modify_roles_ids = util.listify(payload.get('modify_ids[]', None))
-
-        action = payload.get('action', None)
-        if action is None:
-            raise RequestParameterMissingException('The mandatory parameter "action" is missing.')
-        elif action == 'set_permissions':
-
-            # ADD TO LIBRARY ROLES
-            valid_add_roles = []
-            invalid_add_roles_names = []
-            for role_id in new_add_roles_ids:
-                role = self.role_manager.get(trans, trans.security.decode_id(role_id, object_name='role'))
-                #  Check whether role is in the set of allowed roles
-                valid_roles, total_roles = trans.app.security_agent.get_valid_roles(trans, folder)
-                if role in valid_roles:
-                    valid_add_roles.append(role)
-                else:
-                    invalid_add_roles_names.append(role_id)
-            if len(invalid_add_roles_names) > 0:
-                log.warning(f"The following roles could not be added to the add library item permission: {str(invalid_add_roles_names)}")
-
-            # MANAGE FOLDER ROLES
-            valid_manage_roles = []
-            invalid_manage_roles_names = []
-            for role_id in new_manage_roles_ids:
-                role = self.role_manager.get(trans, trans.security.decode_id(role_id, object_name='role'))
-                #  Check whether role is in the set of allowed roles
-                valid_roles, total_roles = trans.app.security_agent.get_valid_roles(trans, folder)
-                if role in valid_roles:
-                    valid_manage_roles.append(role)
-                else:
-                    invalid_manage_roles_names.append(role_id)
-            if len(invalid_manage_roles_names) > 0:
-                log.warning(f"The following roles could not be added to the manage folder permission: {str(invalid_manage_roles_names)}")
-
-            # MODIFY FOLDER ROLES
-            valid_modify_roles = []
-            invalid_modify_roles_names = []
-            for role_id in new_modify_roles_ids:
-                role = self.role_manager.get(trans, trans.security.decode_id(role_id, object_name='role'))
-                #  Check whether role is in the set of allowed roles
-                valid_roles, total_roles = trans.app.security_agent.get_valid_roles(trans, folder)
-                if role in valid_roles:
-                    valid_modify_roles.append(role)
-                else:
-                    invalid_modify_roles_names.append(role_id)
-            if len(invalid_modify_roles_names) > 0:
-                log.warning(f"The following roles could not be added to the modify folder permission: {str(invalid_modify_roles_names)}")
-
-            permissions = {trans.app.security_agent.permitted_actions.LIBRARY_ADD: valid_add_roles}
-            permissions.update({trans.app.security_agent.permitted_actions.LIBRARY_MANAGE: valid_manage_roles})
-            permissions.update({trans.app.security_agent.permitted_actions.LIBRARY_MODIFY: valid_modify_roles})
-
-            trans.app.security_agent.set_all_library_permissions(trans, folder, permissions)
-        else:
-            raise RequestParameterInvalidException('The mandatory parameter "action" has an invalid value.'
-                                                   'Allowed values are: "set_permissions"')
-        current_permissions = self.folder_manager.get_current_roles(trans, folder)
-        return LibraryFolderCurrentPermissions.parse_obj(current_permissions)
-
-    def delete(self, trans, encoded_folder_id: EncodedDatabaseIdField, undelete: Optional[bool] = False) -> LibraryFolderDetails:
-        """
-        DELETE /api/folders/{encoded_folder_id}
-
-        Mark the folder with the given ``encoded_folder_id`` as `deleted`
-        (or remove the `deleted` mark if the `undelete` param is true).
-
-        .. note:: Currently, only admin users can un/delete folders.
-
-        :param  encoded_folder_id:     the encoded id of the folder to un/delete
-        :type   encoded_folder_id:     an encoded id string
-
-        :param  undelete:    (optional) flag specifying whether the item should be deleted or undeleted, defaults to false:
-        :type   undelete:    bool
-
-        :returns:   detailed folder information
-        :rtype:     dictionary
-
-        """
-        folder = self.folder_manager.get(trans, self.folder_manager.cut_and_decode(trans, encoded_folder_id), True)
-        folder = self.folder_manager.delete(trans, folder, undelete)
-        folder_dict = self.folder_manager.get_folder_dict(trans, folder)
-        return LibraryFolderDetails.parse_obj(folder_dict)
-
-    def update(self, trans, encoded_folder_id: EncodedDatabaseIdField, payload: UpdateLibraryFolderPayload) -> LibraryFolderDetails:
-        """
-        Update the folder defined by an ``encoded_folder_id``
-        with the data in the payload.
-
-       .. note:: Currently, only admin users can update library folders. Also the folder must not be `deleted`.
-
-        :param  id:      the encoded id of the folder
-        :type   id:      an encoded id string
-
-        :param  payload: (required) dictionary structure containing::
-            'name':         new folder's name, cannot be empty
-            'description':  new folder's description
-        :type   payload: dict
-
-        :returns:   detailed folder information
-        :rtype:     dict
-
-        :raises: RequestParameterMissingException
-        """
-        decoded_folder_id = self.folder_manager.cut_and_decode(trans, encoded_folder_id)
-        folder = self.folder_manager.get(trans, decoded_folder_id)
-        updated_folder = self.folder_manager.update(trans, folder, payload.name, payload.description)
-        folder_dict = self.folder_manager.get_folder_dict(trans, updated_folder)
-        return LibraryFolderDetails.parse_obj(folder_dict)

lib/galaxy/managers/hdas.py

@@ -505,7 +505,7 @@ class HDASerializer(  # datasets._UnflattenedMetadataDatasetAssociationSerialize
                                 dataset_id=encoded_id, to_ext=hda.extension),
             'report_error': url_for(controller='dataset', action='errors', id=encoded_id),
             'rerun': url_for(controller='tool_runner', action='rerun', id=encoded_id),
-            'show_params': url_for(controller='dataset', action='show_params', dataset_id=encoded_id),
+            'show_params': url_for(controller='dataset', action='details', dataset_id=encoded_id),
             'visualization': url_for(controller='visualization', action='index',
                                      id=encoded_id, model='HistoryDatasetAssociation'),
             'meta_download': url_for(controller='dataset', action='get_metadata_file',

lib/galaxy/managers/histories.py

@@ -4,28 +4,17 @@ Manager and Serializer for histories.
 Histories are containers for datasets or dataset collections
 created (or copied) by users over the course of an analysis.
 """
-import glob
 import logging
-import os
 from typing import (
     cast,
-    List,
     Optional,
     Set,
-    Tuple,
-    Union,
 )
 
-from pydantic import (
-    BaseModel,
-    Field,
-)
 from sqlalchemy import (
     and_,
     asc,
     desc,
-    false,
-    true,
 )
 
 from galaxy import (
@@ -39,65 +28,17 @@ from galaxy.managers import (
     sharable
 )
 from galaxy.managers.base import (
-    ServiceBase,
     SortableManager,
 )
-from galaxy.managers.citations import CitationsManager
-from galaxy.managers.users import UserManager
-from galaxy.schema import FilterQueryParams
-from galaxy.schema.fields import EncodedDatabaseIdField
 from galaxy.schema.schema import (
-    CreateHistoryPayload,
-    CustomBuildsMetadataResponse,
-    ExportHistoryArchivePayload,
-    HistoryBeta,
-    HistoryDetailed,
-    HistoryImportArchiveSourceType,
-    HistorySummary,
-    JobExportHistoryArchive,
-    JobIdResponse,
-    JobImportHistoryResponse,
-    LabelValuePair,
+    HDABasicInfo,
+    ShareHistoryExtra,
 )
-from galaxy.schema.types import SerializationParams
-from galaxy.security.idencoding import IdEncodingHelper
 from galaxy.structured_app import MinimalManagerApp
-from galaxy.util import restore_text
 
 log = logging.getLogger(__name__)
 
 
-class HDABasicInfo(BaseModel):
-    id: EncodedDatabaseIdField
-    name: str
-
-
-class ShareHistoryExtra(sharable.ShareWithExtra):
-    can_change: List[HDABasicInfo] = Field(
-        [],
-        title="Can Change",
-        description=(
-            "A collection of datasets that are not accessible by one or more of the target users "
-            "and that can be made accessible for others by the user sharing the history."
-        ),
-    )
-    cannot_change: List[HDABasicInfo] = Field(
-        [],
-        title="Cannot Change",
-        description=(
-            "A collection of datasets that are not accessible by one or more of the target users "
-            "and that cannot be made accessible for others by the user sharing the history."
-        ),
-    )
-    accessible_count: int = Field(
-        0,
-        title="Accessible Count",
-        description=(
-            "The number of datasets in the history that are public or accessible by all the target users."
-        ),
-    )
-
-
 class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMixin, SortableManager):
 
     model_class = model.History
@@ -248,7 +189,8 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix
         job, _ = history_imp_tool.execute(trans, incoming=incoming)
         return job
 
-    def serve_ready_history_export(self, trans, jeha):
+    # TODO: remove this function when the legacy endpoint using it is removed
+    def legacy_serve_ready_history_export(self, trans, jeha):
         assert jeha.ready
         if jeha.compressed:
             trans.response.set_content_type('application/x-gzip')
@@ -259,6 +201,14 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix
         archive = trans.app.object_store.get_filename(jeha.dataset)
         return open(archive, mode='rb')
 
+    def get_ready_history_export_file_path(self, trans, jeha) -> str:
+        """
+        Serves the history export archive for use as a streaming response so the file
+        doesn't need to be loaded into memory.
+        """
+        assert jeha.ready
+        return trans.app.object_store.get_filename(jeha.dataset)
+
     def queue_history_export(self, trans, history, gzip=True, include_hidden=False, include_deleted=False, directory_uri=None, file_name=None):
         # Convert options to booleans.
         if isinstance(gzip, str):
@@ -363,11 +313,9 @@ class HistoryExportView:
     def serialize(self, trans, history_id, jeha):
         rval = jeha.to_dict()
         encoded_jeha_id = trans.security.encode_id(jeha.id)
-        api_url = self.app.url_for("history_archive_download", id=history_id, jeha_id=encoded_jeha_id)
-        # this URL is less likely to be blocked by a proxy and require an API key, so export
-        # older-style controller version for use with within the GUI and such.
-        external_url = self.app.url_for(controller='history', action="export_archive", id=history_id, qualified=True)
-        external_permanent_url = self.app.url_for(controller='history', action="export_archive", id=history_id, jeha_id=encoded_jeha_id, qualified=True)
+        api_url = trans.url_builder("history_archive_download", id=history_id, jeha_id=encoded_jeha_id)
+        external_url = trans.url_builder("history_archive_download", id=history_id, jeha_id="latest", qualified=True)
+        external_permanent_url = trans.url_builder("history_archive_download", id=history_id, jeha_id=encoded_jeha_id, qualified=True)
         rval["download_url"] = api_url
         rval["external_download_latest_url"] = external_url
         rval["external_download_permanent_url"] = external_permanent_url
@@ -652,425 +600,3 @@ class HistoryFilters(sharable.SharableModelFilters, deletable.PurgableFiltersMix
             'create_time': {'op': ('le', 'ge', 'gt', 'lt'), 'val': self.parse_date},
             'update_time': {'op': ('le', 'ge', 'gt', 'lt'), 'val': self.parse_date},
         })
-
-
-class HistoriesService(ServiceBase):
-    """Common interface/service logic for interactions with histories in the context of the API.
-
-    Provides the logic of the actions invoked by API controllers and uses type definitions
-    and pydantic models to declare its parameters and return types.
-    """
-
-    def __init__(
-        self,
-        security: IdEncodingHelper,
-        manager: HistoryManager,
-        user_manager: UserManager,
-        serializer: HistorySerializer,
-        deserializer: HistoryDeserializer,
-        citations_manager: CitationsManager,
-        history_export_view: HistoryExportView,
-        filters: HistoryFilters,
-    ):
-        super().__init__(security)
-        self.manager = manager
-        self.user_manager = user_manager
-        self.serializer = serializer
-        self.deserializer = deserializer
-        self.citations_manager = citations_manager
-        self.history_export_view = history_export_view
-        self.filters = filters
-        self.shareable_service = sharable.ShareableService(self.manager, self.serializer)
-
-    def index(
-        self,
-        trans,
-        serialization_params: SerializationParams,
-        filter_query_params: FilterQueryParams,
-        deleted_only: Optional[bool] = False,
-        all_histories: bool = False,
-    ):
-        """
-        Return a collection of histories for the current user. Additional filters can be applied.
-
-        :type   deleted_only: optional boolean
-        :param  deleted_only: if True, show only deleted histories, if False, non-deleted
-
-        .. note:: Anonymous users are allowed to get their current history
-        """
-        # bail early with current history if user is anonymous
-        current_user = self.user_manager.current_user(trans)
-        if self.user_manager.is_anonymous(current_user):
-            current_history = self.manager.get_current(trans)
-            if not current_history:
-                return []
-            # note: ignores filters, limit, offset
-            return [self._serialize_history(trans, current_history, serialization_params)]
-
-        filter_params = self.filters.build_filter_params(filter_query_params)
-        filters = []
-        # support the old default of not-returning/filtering-out deleted_only histories
-        filters += self._get_deleted_filter(deleted_only, filter_params)
-
-        # if parameter 'all_histories' is true, throw exception if not admin
-        # else add current user filter to query (default behaviour)
-        if all_histories:
-            if not trans.user_is_admin:
-                message = "Only admins can query all histories"
-                raise glx_exceptions.AdminRequiredException(message)
-        else:
-            filters += [model.History.user == current_user]
-        # and any sent in from the query string
-        filters += self.filters.parse_filters(filter_params)
-        order_by = self.build_order_by(self.manager, filter_query_params.order)
-
-        histories = self.manager.list(
-            filters=filters, order_by=order_by,
-            limit=filter_query_params.limit, offset=filter_query_params.offset)
-
-        rval = [self._serialize_history(trans, history, serialization_params, default_view="summary") for history in histories]
-        return rval
-
-    def _get_deleted_filter(self, deleted: Optional[bool], filter_params: List[Tuple[str, str, str]]):
-        # TODO: this should all be removed (along with the default) in v2
-        # support the old default of not-returning/filtering-out deleted histories
-        try:
-            # the consumer must explicitly ask for both deleted and non-deleted
-            #   but pull it from the parsed params (as the filter system will error on None)
-            deleted_filter_index = filter_params.index(('deleted', 'eq', 'None'))
-            filter_params.pop(deleted_filter_index)
-            return []
-        except ValueError:
-            pass
-
-        # the deleted string bool was also used as an 'include deleted' flag
-        if deleted is True:
-            return [model.History.deleted == true()]
-
-        # the third option not handled here is 'return only deleted'
-        #   if this is passed in (in the form below), simply return and let the filter system handle it
-        if ('deleted', 'eq', 'True') in filter_params:
-            return []
-
-        # otherwise, do the default filter of removing the deleted histories
-        return [model.History.deleted == false()]
-
-    def create(
-        self,
-        trans,
-        payload: CreateHistoryPayload,
-        serialization_params: SerializationParams,
-    ):
-        """Create a new history from scratch, by copying an existing one or by importing
-        from URL or File depending on the provided parameters in the payload.
-        """
-        if trans.user and trans.user.bootstrap_admin_user:
-            raise glx_exceptions.RealUserRequiredException("Only real users can create histories.")
-        hist_name = None
-        if payload.name is not None:
-            hist_name = restore_text(payload.name)
-        copy_this_history_id = payload.history_id
-        all_datasets = payload.all_datasets
-
-        if payload.archive_source is not None:
-            archive_source = payload.archive_source
-            archive_file = payload.archive_file
-            if archive_source:
-                archive_type = payload.archive_type
-            elif archive_file is not None and hasattr(archive_file, "file"):
-                archive_source = archive_file.file.name
-                archive_type = HistoryImportArchiveSourceType.file
-            else:
-                raise glx_exceptions.MessageException("Please provide a url or file.")
-            job = self.manager.queue_history_import(trans, archive_type=archive_type, archive_source=archive_source)
-            job_dict = job.to_dict()
-            job_dict["message"] = f"Importing history from source '{archive_source}'. This history will be visible when the import is complete."
-            job_dict = trans.security.encode_all_ids(job_dict)
-            return JobImportHistoryResponse.parse_obj(job_dict)
-
-        new_history = None
-        # if a history id was passed, copy that history
-        if copy_this_history_id:
-            decoded_id = self.decode_id(copy_this_history_id)
-            original_history = self.manager.get_accessible(decoded_id, trans.user, current_history=trans.history)
-            hist_name = hist_name or (f"Copy of '{original_history.name}'")
-            new_history = original_history.copy(name=hist_name, target_user=trans.user, all_datasets=all_datasets)
-
-        # otherwise, create a new empty history
-        else:
-            new_history = self.manager.create(user=trans.user, name=hist_name)
-
-        trans.app.security_agent.history_set_default_permissions(new_history)
-        trans.sa_session.add(new_history)
-        trans.sa_session.flush()
-
-        # an anonymous user can only have one history
-        if self.user_manager.is_anonymous(trans.user):
-            self.manager.set_current(trans, new_history)
-
-        return self._serialize_history(trans, new_history, serialization_params)
-
-    def show(
-        self,
-        trans,
-        serialization_params: SerializationParams,
-        history_id: Optional[EncodedDatabaseIdField] = None,
-    ):
-        """
-        Returns detailed information about the history with the given encoded `id`. If no `id` is
-        provided, then the most recently used history will be returned.
-
-        :type   id:      an optional encoded id string
-        :param  id:      the encoded id of the history to query or None to use the most recently used
-
-        :type   serialization_params:   dictionary
-        :param  serialization_params:   contains the optional `view`, `keys` and `default_view` for serialization
-
-        :rtype:     dictionary
-        :returns:   detailed history information
-        """
-        if history_id is None:  # By default display the most recent history
-            history = self.manager.most_recent(
-                trans.user,
-                filters=(model.History.deleted == false()),
-                current_history=trans.history
-            )
-        else:
-            history = self.manager.get_accessible(
-                self.decode_id(history_id),
-                trans.user,
-                current_history=trans.history
-            )
-        return self._serialize_history(trans, history, serialization_params)
-
-    def update(
-        self,
-        trans,
-        id: EncodedDatabaseIdField,
-        payload,
-        serialization_params: SerializationParams,
-    ):
-        """Updates the values for the history with the given ``id``
-
-        :type   id:      str
-        :param  id:      the encoded id of the history to update
-        :type   payload: dict
-        :param  payload: a dictionary containing any or all the
-            fields in :func:`galaxy.model.History.to_dict` and/or the following:
-
-            * annotation: an annotation for the history
-
-        :type   serialization_params:   dictionary
-        :param  serialization_params:   contains the optional `view`, `keys` and `default_view` for serialization
-
-        :rtype:     dict
-        :returns:   an error object if an error occurred or a dictionary containing
-            any values that were different from the original and, therefore, updated
-        """
-        # TODO: PUT /api/histories/{encoded_history_id} payload = { rating: rating } (w/ no security checks)
-        history = self.manager.get_owned(self.decode_id(id), trans.user, current_history=trans.history)
-        self.deserializer.deserialize(history, payload, user=trans.user, trans=trans)
-        return self._serialize_history(trans, history, serialization_params)
-
-    def delete(
-        self,
-        trans,
-        history_id: EncodedDatabaseIdField,
-        serialization_params: SerializationParams,
-        purge: bool = False,
-    ):
-        """Delete the history with the given ``id``
-
-        .. note:: Stops all active jobs in the history if purge is set.
-
-        You can purge a history, removing all it's datasets from disk (if unshared),
-        by passing in ``purge=True`` in the url.
-
-        :type   serialization_params:   dictionary
-        :param  serialization_params:   contains the optional `view`, `keys` and `default_view` for serialization
-
-        :rtype:     dict
-        :returns:   the deleted or purged history
-        """
-        history = self.manager.get_owned(self.decode_id(history_id), trans.user, current_history=trans.history)
-        if purge:
-            self.manager.purge(history)
-        else:
-            self.manager.delete(history)
-        return self._serialize_history(trans, history, serialization_params)
-
-    def undelete(
-        self,
-        trans,
-        history_id: EncodedDatabaseIdField,
-        serialization_params: SerializationParams,
-    ):
-        """Undelete history (that hasn't been purged) with the given ``id``
-
-        :type   id:     str
-        :param  id:     the encoded id of the history to undelete
-
-        :type   serialization_params:   dictionary
-        :param  serialization_params:   contains the optional `view`, `keys` and `default_view` for serialization
-
-        :rtype:     dict
-        :returns:   the undeleted history
-        """
-        history = self.manager.get_owned(self.decode_id(history_id), trans.user, current_history=trans.history)
-        self.manager.undelete(history)
-        return self._serialize_history(trans, history, serialization_params)
-
-    def shared_with_me(
-        self,
-        trans,
-        serialization_params: SerializationParams,
-        filter_query_params: FilterQueryParams,
-    ):
-        """
-        Return all histories that are shared with the current user. The results can be filtered.
-        """
-        current_user = trans.user
-        filters = self.filters.parse_query_filters(filter_query_params)
-        order_by = self.build_order_by(self.manager, filter_query_params.order)
-        histories = self.manager.list_shared_with(current_user,
-            filters=filters, order_by=order_by,
-            limit=filter_query_params.limit, offset=filter_query_params.offset)
-        rval = [self._serialize_history(trans, history, serialization_params, default_view="summary") for history in histories]
-        return rval
-
-    def published(
-        self,
-        trans,
-        serialization_params: SerializationParams,
-        filter_query_params: FilterQueryParams,
-    ):
-        """
-        Return all histories that are published. The results can be filtered.
-        """
-        filters = self.filters.parse_query_filters(filter_query_params)
-        order_by = self.build_order_by(self.manager, filter_query_params.order)
-        histories = self.manager.list_published(
-            filters=filters, order_by=order_by,
-            limit=filter_query_params.limit, offset=filter_query_params.offset,
-        )
-        rval = [self._serialize_history(trans, history, serialization_params, default_view="summary") for history in histories]
-        return rval
-
-    def citations(self, trans, history_id):
-        """
-        Return all the citations for the tools used to produce the datasets in
-        the history.
-        """
-        history = self.manager.get_accessible(self.decode_id(history_id), trans.user, current_history=trans.history)
-        tool_ids = set()
-        for dataset in history.datasets:
-            job = dataset.creating_job
-            if not job:
-                continue
-            tool_id = job.tool_id
-            if not tool_id:
-                continue
-            tool_ids.add(tool_id)
-        return [citation.to_dict("bibtex") for citation in self.citations_manager.citations_for_tool_ids(tool_ids)]
-
-    def index_exports(self, trans, id):
-        """
-        Get previous history exports (to links). Effectively returns serialized
-        JEHA objects.
-        """
-        return self.history_export_view.get_exports(trans, id)
-
-    def archive_export(self, trans, id: EncodedDatabaseIdField, payload: ExportHistoryArchivePayload) -> Union[JobExportHistoryArchive, JobIdResponse]:
-        """
-        start job (if needed) to create history export for corresponding
-        history.
-
-        :type   id:     str
-        :param  id:     the encoded id of the history to export
-
-        :rtype:     dict
-        :returns:   object containing url to fetch export from.
-        """
-        history = self.manager.get_accessible(self.decode_id(id), trans.user, current_history=trans.history)
-        jeha = history.latest_export
-        exporting_to_uri = payload.directory_uri
-        # always just issue a new export when exporting to a URI.
-        up_to_date = not payload.force and not exporting_to_uri and (jeha and jeha.up_to_date)
-        job = None
-        if not up_to_date:
-            # Need to create new JEHA + job.
-            job = self.manager.queue_history_export(
-                trans,
-                history,
-                gzip=payload.gzip,
-                include_hidden=payload.include_hidden,
-                include_deleted=payload.include_deleted,
-                directory_uri=payload.directory_uri,
-                file_name=payload.file_name,
-            )
-        else:
-            job = jeha.job
-
-        if exporting_to_uri:
-            # we don't have a jeha, there will never be a download_url. Just let
-            # the client poll on the created job_id to determine when the file has been
-            # written.
-            job_id = trans.security.encode_id(job.id)
-            return JobIdResponse(job_id=job_id)
-
-        if up_to_date and jeha.ready:
-            serialized_jeha = self.history_export_view.serialize(trans, id, jeha)
-            return JobExportHistoryArchive.parse_obj(serialized_jeha)
-        else:
-            # Valid request, just resource is not ready yet.
-            trans.response.status = "202 Accepted"
-            if jeha:
-                serialized_jeha = self.history_export_view.serialize(trans, id, jeha)
-                return JobExportHistoryArchive.parse_obj(serialized_jeha)
-            else:
-                assert job is not None, "logic error, don't have a jeha or a job"
-                job_id = trans.security.encode_id(job.id)
-                return JobIdResponse(job_id=job_id)
-
-    def archive_download(self, trans, id, jeha_id):
-        """
-        If ready and available, return raw contents of exported history.
-        """
-        jeha = self.history_export_view.get_ready_jeha(trans, id, jeha_id)
-        return self.manager.serve_ready_history_export(trans, jeha)
-
-    def get_custom_builds_metadata(self, trans, id: EncodedDatabaseIdField) -> CustomBuildsMetadataResponse:
-        """
-        Returns meta data for custom builds.
-        """
-        history = self.manager.get_accessible(self.decode_id(id), trans.user, current_history=trans.history)
-        installed_builds = []
-        for build in glob.glob(os.path.join(trans.app.config.len_file_path, "*.len")):
-            installed_builds.append(os.path.basename(build).split(".len")[0])
-        fasta_hdas = trans.sa_session.query(model.HistoryDatasetAssociation) \
-            .filter_by(history=history, extension="fasta", deleted=False) \
-            .order_by(model.HistoryDatasetAssociation.hid.desc())
-        return CustomBuildsMetadataResponse(
-            installed_builds=[LabelValuePair(label=ins, value=ins) for ins in installed_builds],
-            fasta_hdas=[LabelValuePair(label=f'{hda.hid}: {hda.name}', value=trans.security.encode_id(hda.id)) for hda in fasta_hdas],
-        )
-
-    def _serialize_history(
-            self,
-            trans,
-            history: model.History,
-            serialization_params: SerializationParams,
-            default_view: str = "detailed",
-    ) -> Union[HistoryBeta, HistoryDetailed, HistorySummary]:
-        """
-        Returns a dictionary with the corresponding values depending on the
-        serialization parameters provided.
-        """
-        serialization_params["default_view"] = default_view
-        serialized_history = self.serializer.serialize_to_view(
-            history,
-            user=trans.user,
-            trans=trans,
-            **serialization_params
-        )
-        return serialized_history

lib/galaxy/managers/history_contents.py

@@ -411,8 +411,8 @@ class HistoryContentsManager(containers.ContainerManagerMixin, base.SortableMana
         # This will conditionally join a potentially costly job_state summary
         # All the paranoia if-checking makes me wonder if serialization_params
         # should really be a property of the manager class instance
-        if serialization_params and serialization_params['keys']:
-            if 'job_state_summary' in serialization_params['keys']:
+        if serialization_params and serialization_params.keys:
+            if 'job_state_summary' in serialization_params.keys:
                 query = query.options(eagerload('job_state_summary'))
 
         return {row.id: row for row in query.all()}

lib/galaxy/managers/libraries.py

@@ -3,10 +3,7 @@ Manager and Serializer for libraries.
 """
 import logging
 from typing import (
-    Any,
-    Dict,
     Optional,
-    Union,
 )
 
 from sqlalchemy import and_, false, not_, or_, true
@@ -15,22 +12,8 @@ from sqlalchemy.orm.exc import NoResultFound
 
 from galaxy import (
     exceptions,
-    util,
 )
-from galaxy.managers.context import ProvidesAppContext
 from galaxy.managers.folders import FolderManager
-from galaxy.managers.roles import RoleManager
-from galaxy.schema.fields import EncodedDatabaseIdField
-from galaxy.schema.schema import (
-    CreateLibraryPayload,
-    LibraryAvailablePermissions,
-    LibraryCurrentPermissions,
-    LibraryLegacySummary,
-    LibraryPermissionScope,
-    LibrarySummary,
-    LibrarySummaryList,
-    UpdateLibraryPayload,
-)
 from galaxy.util import (
     pretty_print_time_interval,
     unicodify,
@@ -329,313 +312,3 @@ def get_containing_library_from_library_dataset(trans, library_dataset):
         if library.root_folder == folder:
             return library
     return None
-
-
-class LibrariesService:
-    """
-    Interface/service object for sharing logic between controllers.
-    """
-
-    def __init__(
-        self,
-        folder_manager: FolderManager,
-        library_manager: LibraryManager,
-        role_manager: RoleManager,
-    ):
-        self.folder_manager = folder_manager
-        self.library_manager = library_manager
-        self.role_manager = role_manager
-
-    def index(self, trans: ProvidesAppContext, deleted: Optional[bool] = False) -> LibrarySummaryList:
-        """Returns a list of summary data for all libraries.
-
-        :param  deleted: if True, show only ``deleted`` libraries, if False show only ``non-deleted``
-        :type   deleted: boolean (optional)
-
-        :returns:   list of dictionaries containing library information
-        :rtype:     list
-
-        .. seealso:: :attr:`galaxy.model.Library.dict_collection_visible_keys`
-
-        """
-        query, prefetched_ids = self.library_manager.list(trans, deleted)
-        libraries = []
-        for library in query:
-            libraries.append(self.library_manager.get_library_dict(trans, library, prefetched_ids))
-        return LibrarySummaryList.parse_obj(libraries)
-
-    def show(self, trans, id: EncodedDatabaseIdField) -> LibrarySummary:
-        """ Returns detailed information about a library.
-
-        :param  id:      the encoded id of the library
-        :type   id:      an encoded id string
-        :param  deleted: if True, allow information on a ``deleted`` library
-        :type   deleted: boolean
-
-        :returns:   detailed library information
-        :rtype:     dict
-
-        .. seealso:: :attr:`galaxy.model.Library.dict_element_visible_keys`
-
-        :raises: MalformedId, ObjectNotFound
-        """
-        library = self.library_manager.get(trans, trans.security.decode_id(id, object_name='library'))
-        library_dict = self.library_manager.get_library_dict(trans, library)
-        return LibrarySummary.parse_obj(library_dict)
-
-    def create(self, trans, payload: CreateLibraryPayload) -> LibrarySummary:
-        """Creates a new library.
-
-        .. note:: Currently, only admin users can create libraries.
-
-        :param  payload: dictionary structure containing::
-            :param name:         (required) the new library's name
-            :type  name:         str
-            :param description:  the new library's description
-            :type  description:  str
-            :param synopsis:     the new library's synopsis
-            :type  synopsis:     str
-        :type   payload: dict
-        :returns:   detailed library information
-        :rtype:     dict
-        :raises: RequestParameterMissingException
-        """
-        library = self.library_manager.create(trans, payload.name, payload.description, payload.synopsis)
-        library_dict = self.library_manager.get_library_dict(trans, library)
-        return LibrarySummary.parse_obj(library_dict)
-
-    def update(self, trans, id: EncodedDatabaseIdField, payload: UpdateLibraryPayload) -> LibrarySummary:
-        """Updates the library defined by an ``encoded_id`` with the data in the payload.
-
-       .. note:: Currently, only admin users can update libraries. Also the library must not be `deleted`.
-
-        :param  id:      the encoded id of the library
-        :type   id:      an encoded id string
-        :param  payload: dictionary structure containing::
-            :param name:         new library's name, cannot be empty
-            :type  name:         str
-            :param description:  new library's description
-            :type  description:  str
-            :param synopsis:     new library's synopsis
-            :type  synopsis:     str
-        :type   payload: dict
-        :returns:   detailed library information
-        :rtype:     dict
-        :raises: RequestParameterMissingException
-        """
-        library = self.library_manager.get(trans, trans.security.decode_id(id, object_name='library'))
-        name = payload.name
-        if name == '':
-            raise exceptions.RequestParameterMissingException("Parameter 'name' of library is required. You cannot remove it.")
-        updated_library = self.library_manager.update(trans, library, name, payload.description, payload.synopsis)
-        library_dict = self.library_manager.get_library_dict(trans, updated_library)
-        return LibrarySummary.parse_obj(library_dict)
-
-    def delete(self, trans, id: EncodedDatabaseIdField, undelete: Optional[bool] = False) -> LibrarySummary:
-        """Marks the library with the given ``id`` as `deleted` (or removes the `deleted` mark if the `undelete` param is true)
-
-        .. note:: Currently, only admin users can un/delete libraries.
-
-        :param  id:     the encoded id of the library to un/delete
-        :type   id:     an encoded id string
-
-        :param  undelete:    (optional) flag specifying whether the item should be deleted or undeleted, defaults to false:
-        :type   undelete:    bool
-
-        :returns:   detailed library information
-        :rtype:     dictionary
-
-        .. seealso:: :attr:`galaxy.model.Library.dict_element_visible_keys`
-        """
-        library = self.library_manager.get(trans, trans.security.decode_id(id, object_name='library'))
-        library = self.library_manager.delete(trans, library, undelete)
-        library_dict = self.library_manager.get_library_dict(trans, library)
-        return LibrarySummary.parse_obj(library_dict)
-
-    def get_permissions(
-        self,
-        trans,
-        id: EncodedDatabaseIdField,
-        scope: Optional[LibraryPermissionScope] = LibraryPermissionScope.current,
-        is_library_access: Optional[bool] = False,
-        page: Optional[int] = 1,
-        page_limit: Optional[int] = 10,
-        query: Optional[str] = None,
-    ) -> Union[LibraryCurrentPermissions, LibraryAvailablePermissions]:
-        """Load all permissions for the given library id and return it.
-
-        :param  id:     the encoded id of the library
-        :type   id:     an encoded id string
-
-        :param  scope:      either 'current' or 'available'
-        :type   scope:      string
-
-        :param  is_library_access:      indicates whether the roles available for the library access are requested
-        :type   is_library_access:      bool
-
-        :returns:   dictionary with all applicable permissions' values
-        :rtype:     dictionary
-
-        :raises: InsufficientPermissionsException
-        """
-        current_user_roles = trans.get_current_user_roles()
-        is_admin = trans.user_is_admin
-        library = self.library_manager.get(trans, trans.security.decode_id(id, object_name='library'))
-        if not (is_admin or trans.app.security_agent.can_manage_library_item(current_user_roles, library)):
-            raise exceptions.InsufficientPermissionsException('You do not have proper permission to access permissions of this library.')
-
-        if scope == LibraryPermissionScope.current or scope is None:
-            roles = self.library_manager.get_current_roles(trans, library)
-            return LibraryCurrentPermissions.parse_obj(roles)
-
-        #  Return roles that are available to select.
-        elif scope == LibraryPermissionScope.available:
-            roles, total_roles = trans.app.security_agent.get_valid_roles(trans, library, query, page, page_limit, is_library_access)
-
-            return_roles = []
-            for role in roles:
-                role_id = trans.security.encode_id(role.id)
-                return_roles.append(dict(id=role_id, name=role.name, type=role.type))
-            return LibraryAvailablePermissions(roles=return_roles, page=page, page_limit=page_limit, total=total_roles)
-        else:
-            raise exceptions.RequestParameterInvalidException("The value of 'scope' parameter is invalid. Alllowed values: current, available")
-
-    def set_permissions(
-        self, trans, id: EncodedDatabaseIdField, payload: Dict[str, Any]
-    ) -> Union[
-        LibraryLegacySummary,  # Old legacy response
-        LibraryCurrentPermissions,
-    ]:
-        """Set permissions of the given library to the given role ids.
-
-        :param  id:      the encoded id of the library to set the permissions of
-        :type   id:      an encoded id string
-        :param   payload: dictionary structure containing:
-
-            :param  action:            (required) describes what action should be performed
-                                       available actions: remove_restrictions, set_permissions
-            :type   action:            str
-            :param  access_ids[]:      list of Role.id defining roles that should have access permission on the library
-            :type   access_ids[]:      string or list
-            :param  add_ids[]:         list of Role.id defining roles that should have add item permission on the library
-            :type   add_ids[]:         string or list
-            :param  manage_ids[]:      list of Role.id defining roles that should have manage permission on the library
-            :type   manage_ids[]:      string or list
-            :param  modify_ids[]:      list of Role.id defining roles that should have modify permission on the library
-            :type   modify_ids[]:      string or list
-
-        :type:      dictionary
-        :returns:   dict of current roles for all available permission types
-        :rtype:     dictionary
-        :raises: RequestParameterInvalidException, InsufficientPermissionsException, InternalServerError
-                    RequestParameterMissingException
-        """
-        is_admin = trans.user_is_admin
-        current_user_roles = trans.get_current_user_roles()
-        library = self.library_manager.get(trans, trans.security.decode_id(id, object_name='library'))
-
-        if not (is_admin or trans.app.security_agent.can_manage_library_item(current_user_roles, library)):
-            raise exceptions.InsufficientPermissionsException('You do not have proper permission to modify permissions of this library.')
-
-        new_access_roles_ids = util.listify(payload.get('access_ids[]', None))
-        new_add_roles_ids = util.listify(payload.get('add_ids[]', None))
-        new_manage_roles_ids = util.listify(payload.get('manage_ids[]', None))
-        new_modify_roles_ids = util.listify(payload.get('modify_ids[]', None))
-
-        action = payload.get('action', None)
-        if action is None:
-            if payload is not None:
-                return self.set_permissions_old(trans, library, payload)
-            else:
-                raise exceptions.RequestParameterMissingException('The mandatory parameter "action" is missing.')
-        elif action == 'remove_restrictions':
-            is_public = self.library_manager.make_public(trans, library)
-            if not is_public:
-                raise exceptions.InternalServerError('An error occurred while making library public.')
-        elif action == 'set_permissions':
-
-            # ACCESS LIBRARY ROLES
-            valid_access_roles = []
-            invalid_access_roles_names = []
-            for role_id in new_access_roles_ids:
-                role = self.role_manager.get(trans, trans.security.decode_id(role_id, object_name='role'))
-                valid_roles, total_roles = trans.app.security_agent.get_valid_roles(trans, library, is_library_access=True)
-                if role in valid_roles:
-                    valid_access_roles.append(role)
-                else:
-                    invalid_access_roles_names.append(role_id)
-            if len(invalid_access_roles_names) > 0:
-                log.warning(f"The following roles could not be added to the library access permission: {str(invalid_access_roles_names)}")
-
-            # ADD TO LIBRARY ROLES
-            valid_add_roles = []
-            invalid_add_roles_names = []
-            for role_id in new_add_roles_ids:
-                role = self.role_manager.get(trans, trans.security.decode_id(role_id, object_name='role'))
-                valid_roles, total_roles = trans.app.security_agent.get_valid_roles(trans, library)
-                if role in valid_roles:
-                    valid_add_roles.append(role)
-                else:
-                    invalid_add_roles_names.append(role_id)
-            if len(invalid_add_roles_names) > 0:
-                log.warning(f"The following roles could not be added to the add library item permission: {str(invalid_add_roles_names)}")
-
-            # MANAGE LIBRARY ROLES
-            valid_manage_roles = []
-            invalid_manage_roles_names = []
-            for role_id in new_manage_roles_ids:
-                role = self.role_manager.get(trans, trans.security.decode_id(role_id, object_name='role'))
-                valid_roles, total_roles = trans.app.security_agent.get_valid_roles(trans, library)
-                if role in valid_roles:
-                    valid_manage_roles.append(role)
-                else:
-                    invalid_manage_roles_names.append(role_id)
-            if len(invalid_manage_roles_names) > 0:
-                log.warning(f"The following roles could not be added to the manage library permission: {str(invalid_manage_roles_names)}")
-
-            # MODIFY LIBRARY ROLES
-            valid_modify_roles = []
-            invalid_modify_roles_names = []
-            for role_id in new_modify_roles_ids:
-                role = self.role_manager.get(trans, trans.security.decode_id(role_id, object_name='role'))
-                valid_roles, total_roles = trans.app.security_agent.get_valid_roles(trans, library)
-                if role in valid_roles:
-                    valid_modify_roles.append(role)
-                else:
-                    invalid_modify_roles_names.append(role_id)
-            if len(invalid_modify_roles_names) > 0:
-                log.warning(f"The following roles could not be added to the modify library permission: {str(invalid_modify_roles_names)}")
-
-            permissions = {trans.app.security_agent.permitted_actions.LIBRARY_ACCESS: valid_access_roles}
-            permissions.update({trans.app.security_agent.permitted_actions.LIBRARY_ADD: valid_add_roles})
-            permissions.update({trans.app.security_agent.permitted_actions.LIBRARY_MANAGE: valid_manage_roles})
-            permissions.update({trans.app.security_agent.permitted_actions.LIBRARY_MODIFY: valid_modify_roles})
-
-            trans.app.security_agent.set_all_library_permissions(trans, library, permissions)
-            trans.sa_session.refresh(library)
-            # Copy the permissions to the root folder
-            trans.app.security_agent.copy_library_permissions(trans, library, library.root_folder)
-        else:
-            raise exceptions.RequestParameterInvalidException('The mandatory parameter "action" has an invalid value.'
-                                                              'Allowed values are: "remove_restrictions", set_permissions"')
-        roles = self.library_manager.get_current_roles(trans, library)
-        return LibraryCurrentPermissions.parse_obj(roles)
-
-    def set_permissions_old(self, trans, library, payload) -> LibraryLegacySummary:
-        """
-        *** old implementation for backward compatibility ***
-
-        Updates the library permissions.
-        """
-        params = util.Params(payload)
-        permissions = {}
-        for k, v in trans.app.model.Library.permitted_actions.items():
-            role_params = params.get(f"{k}_in", [])
-            in_roles = [trans.sa_session.query(trans.app.model.Role).get(trans.security.decode_id(x)) for x in util.listify(role_params)]
-            permissions[trans.app.security_agent.get_action(v.action)] = in_roles
-        trans.app.security_agent.set_all_library_permissions(trans, library, permissions)
-        trans.sa_session.refresh(library)
-        # Copy the permissions to the root folder
-        trans.app.security_agent.copy_library_permissions(trans, library, library.root_folder)
-        item = library.to_dict(view='element', value_mapper={'id': trans.security.encode_id, 'root_folder_id': trans.security.encode_id})
-        return LibraryLegacySummary.parse_obj(item)

lib/galaxy/managers/pages.py

@@ -7,31 +7,21 @@ from within Galaxy.
 """
 import logging
 import re
-from enum import Enum
 from html.entities import name2codepoint
 from html.parser import HTMLParser
 from typing import (
     Callable,
-    List,
-    Optional,
-)
-
-from pydantic import (
-    BaseModel,
-    Extra,
-    Field,
 )
 
 from galaxy import exceptions, model
 from galaxy.managers import base, sharable
 from galaxy.managers.context import ProvidesHistoryContext
 from galaxy.managers.markdown_util import (
-    internal_galaxy_markdown_to_pdf,
     ready_galaxy_markdown_for_export,
     ready_galaxy_markdown_for_import,
 )
 from galaxy.model.item_attrs import UsesAnnotations
-from galaxy.schema.fields import EncodedDatabaseIdField
+from galaxy.schema.schema import PageContentFormat
 from galaxy.structured_app import MinimalManagerApp
 from galaxy.util import unicodify
 from galaxy.util.sanitize_html import sanitize_html
@@ -70,223 +60,6 @@ _cp1252 = {
 }
 
 
-class PageContentFormat(str, Enum):
-    markdown = "markdown"
-    html = "html"
-
-
-ContentFormatField: PageContentFormat = Field(
-    default=PageContentFormat.html,
-    title="Content format",
-    description="Either `markdown` or `html`.",
-)
-
-ContentField: Optional[str] = Field(
-    default="",
-    title="Content",
-    description="Raw text contents of the first page revision (type dependent on content_format).",
-)
-
-
-class PageSummaryBase(BaseModel):
-    title: str = Field(
-        ...,  # Required
-        title="Title",
-        description="The name of the page",
-    )
-    slug: str = Field(
-        ...,  # Required
-        title="Identifier",
-        description="The title slug for the page URL, must be unique.",
-        regex=r"^[a-z0-9\-]+$",
-    )
-
-
-class CreatePagePayload(PageSummaryBase):
-    content_format: PageContentFormat = ContentFormatField
-    content: Optional[str] = ContentField
-    annotation: Optional[str] = Field(
-        default=None,
-        title="Annotation",
-        description="Annotation that will be attached to the page.",
-    )
-    invocation_id: Optional[EncodedDatabaseIdField] = Field(
-        None,
-        title="Workflow invocation ID",
-        description="Encoded ID used by workflow generated reports.",
-    )
-
-    class Config:
-        use_enum_values = True  # When using .dict()
-        extra = Extra.allow  # Allow any other extra fields
-
-
-class PageSummary(PageSummaryBase):
-    id: EncodedDatabaseIdField = Field(
-        ...,  # Required
-        title="ID",
-        description="Encoded ID of the Page.",
-    )
-    model_class: str = Field(
-        ...,  # Required
-        title="Model class",
-        description="The class of the model associated with the ID.",
-        example="Page",
-    )
-    username: str = Field(
-        ...,  # Required
-        title="Username",
-        description="The name of the user owning this Page.",
-    )
-    published: bool = Field(
-        ...,  # Required
-        title="Published",
-        description="Whether this Page has been published.",
-    )
-    importable: bool = Field(
-        ...,  # Required
-        title="Importable",
-        description="Whether this Page can be imported.",
-    )
-    deleted: bool = Field(
-        ...,  # Required
-        title="Deleted",
-        description="Whether this Page has been deleted.",
-    )
-    latest_revision_id: EncodedDatabaseIdField = Field(
-        ...,  # Required
-        title="Latest revision ID",
-        description="The encoded ID of the last revision of this Page.",
-    )
-    revision_ids: List[EncodedDatabaseIdField] = Field(
-        ...,  # Required
-        title="List of revisions",
-        description="The history with the encoded ID of each revision of the Page.",
-    )
-
-
-class PageDetails(PageSummary):
-    content_format: PageContentFormat = ContentFormatField
-    content: Optional[str] = ContentField
-    generate_version: Optional[str] = Field(
-        None,
-        title="Galaxy Version",
-        description="The version of Galaxy this page was generated with.",
-    )
-    generate_time: Optional[str] = Field(
-        None,
-        title="Generate Date",
-        description="The date this page was generated.",
-    )
-
-    class Config:
-        extra = Extra.allow  # Allow any other extra fields
-
-
-class PageSummaryList(BaseModel):
-    __root__: List[PageSummary] = Field(
-        default=[],
-        title='List with summary information of Pages.',
-    )
-
-
-class PagesService:
-    """Common interface/service logic for interactions with pages in the context of the API.
-
-    Provides the logic of the actions invoked by API controllers and uses type definitions
-    and pydantic models to declare its parameters and return types.
-    """
-
-    def __init__(self, app: MinimalManagerApp):
-        self.manager = PageManager(app)
-        self.serializer = PageSerializer(app)
-        self.shareable_service = sharable.ShareableService(self.manager, self.serializer)
-
-    def index(self, trans, deleted: bool = False) -> PageSummaryList:
-        """Return a list of Pages viewable by the user
-
-        :param deleted: Display deleted pages
-
-        :rtype:     list
-        :returns:   dictionaries containing summary or detailed Page information
-        """
-        out = []
-
-        if trans.user_is_admin:
-            r = trans.sa_session.query(model.Page)
-            if not deleted:
-                r = r.filter_by(deleted=False)
-            for row in r:
-                out.append(trans.security.encode_all_ids(row.to_dict(), recursive=True))
-        else:
-            # Transaction user's pages (if any)
-            user = trans.user
-            r = trans.sa_session.query(model.Page).filter_by(user=user)
-            if not deleted:
-                r = r.filter_by(deleted=False)
-            for row in r:
-                out.append(trans.security.encode_all_ids(row.to_dict(), recursive=True))
-            # Published pages from other users
-            r = trans.sa_session.query(model.Page).filter(model.Page.user != user).filter_by(published=True)
-            if not deleted:
-                r = r.filter_by(deleted=False)
-            for row in r:
-                out.append(trans.security.encode_all_ids(row.to_dict(), recursive=True))
-
-        return PageSummaryList.parse_obj(out)
-
-    def create(self, trans, payload: CreatePagePayload) -> PageSummary:
-        """
-        Create a page and return Page summary
-        """
-        page = self.manager.create(trans, payload.dict())
-        rval = trans.security.encode_all_ids(page.to_dict(), recursive=True)
-        rval['content'] = page.latest_revision.content
-        self.manager.rewrite_content_for_export(trans, rval)
-        return PageSummary.parse_obj(rval)
-
-    def delete(self, trans, id: EncodedDatabaseIdField):
-        """
-        Deletes a page (or marks it as deleted)
-        """
-        page = base.get_object(trans, id, 'Page', check_ownership=True)
-
-        # Mark a page as deleted
-        page.deleted = True
-        trans.sa_session.flush()
-
-    def show(self, trans, id: EncodedDatabaseIdField) -> PageDetails:
-        """View a page summary and the content of the latest revision
-
-        :param  id:    ID of page to be displayed
-
-        :rtype:     dict
-        :returns:   Dictionary return of the Page.to_dict call with the 'content' field populated by the most recent revision
-        """
-        page = base.get_object(trans, id, 'Page', check_ownership=False, check_accessible=True)
-        rval = trans.security.encode_all_ids(page.to_dict(), recursive=True)
-        rval['content'] = page.latest_revision.content
-        rval['content_format'] = page.latest_revision.content_format
-        self.manager.rewrite_content_for_export(trans, rval)
-        return PageDetails.parse_obj(rval)
-
-    def show_pdf(self, trans, id: EncodedDatabaseIdField):
-        """
-        View a page summary and the content of the latest revision as PDF.
-
-        :param  id: ID of page to be displayed
-
-        :rtype: dict
-        :returns: Dictionary return of the Page.to_dict call with the 'content' field populated by the most recent revision
-        """
-        page = base.get_object(trans, id, 'Page', check_ownership=False, check_accessible=True)
-        if page.latest_revision.content_format != PageContentFormat.markdown.value:
-            raise exceptions.RequestParameterInvalidException("PDF export only allowed for Markdown based pages")
-        internal_galaxy_markdown = page.latest_revision.content
-        trans.response.set_content_type("application/pdf")
-        return internal_galaxy_markdown_to_pdf(trans, internal_galaxy_markdown, 'page')
-
-
 class PageManager(sharable.SharableModelManager, UsesAnnotations):
     """Provides operations for managing a Page."""
 

lib/galaxy/managers/quotas.py

@@ -11,29 +11,17 @@ from typing import (
     Union,
 )
 
-from sqlalchemy import (
-    false,
-    true
-)
-
 from galaxy import model, util
 from galaxy.app import StructuredApp
 from galaxy.exceptions import ActionInputError
 from galaxy.managers import base
-from galaxy.managers.context import ProvidesUserContext
 from galaxy.quota import DatabaseQuotaAgent
 from galaxy.quota._schema import (
     CreateQuotaParams,
-    CreateQuotaResult,
     DefaultQuotaValues,
-    DeleteQuotaPayload,
-    QuotaDetails,
     QuotaOperation,
-    QuotaSummaryList,
-    UpdateQuotaParams,
 )
 from galaxy.schema.fields import EncodedDatabaseIdField
-from galaxy.web import url_for
 
 log = logging.getLogger(__name__)
 
@@ -247,119 +235,3 @@ class QuotaManager:
 
     def get_quota(self, trans, id: EncodedDatabaseIdField, deleted: Optional[bool] = None) -> model.Quota:
         return base.get_object(trans, id, 'Quota', check_ownership=False, check_accessible=False, deleted=deleted)
-
-
-class QuotasService:
-    """Interface/service object shared by controllers for interacting with quotas."""
-
-    def __init__(self, app: StructuredApp):
-        self.quota_manager: QuotaManager = QuotaManager(app)
-
-    def index(self, trans: ProvidesUserContext, deleted: bool = False) -> QuotaSummaryList:
-        """Displays a collection (list) of quotas."""
-        rval = []
-        query = trans.sa_session.query(model.Quota)
-        if deleted:
-            route = 'deleted_quota'
-            query = query.filter(model.Quota.deleted == true())
-        else:
-            route = 'quota'
-            query = query.filter(model.Quota.deleted == false())
-        for quota in query:
-            item = quota.to_dict(value_mapper={'id': trans.security.encode_id})
-            encoded_id = trans.security.encode_id(quota.id)
-            item['url'] = self._url_for(route, id=encoded_id)
-            rval.append(item)
-        return QuotaSummaryList.parse_obj(rval)
-
-    def show(self, trans: ProvidesUserContext, id: EncodedDatabaseIdField, deleted: bool = False) -> QuotaDetails:
-        """Displays information about a quota."""
-        quota = self.quota_manager.get_quota(trans, id, deleted=deleted)
-        rval = quota.to_dict(view='element', value_mapper={'id': trans.security.encode_id, 'total_disk_usage': float})
-        return QuotaDetails.parse_obj(rval)
-
-    def create(self, trans: ProvidesUserContext, params: CreateQuotaParams) -> CreateQuotaResult:
-        """Creates a new quota."""
-        payload = params.dict()
-        self.validate_in_users_and_groups(trans, payload)
-        quota, message = self.quota_manager.create_quota(payload)
-        item = quota.to_dict(value_mapper={'id': trans.security.encode_id})
-        item['url'] = self._url_for('quota', id=trans.security.encode_id(quota.id))
-        item['message'] = message
-        return CreateQuotaResult.parse_obj(item)
-
-    def update(self, trans: ProvidesUserContext, id: EncodedDatabaseIdField, params: UpdateQuotaParams) -> str:
-        """Modifies a quota."""
-        payload = params.dict()
-        self.validate_in_users_and_groups(trans, payload)
-        quota = self.quota_manager.get_quota(trans, id, deleted=False)
-
-        params = UpdateQuotaParams(**payload)
-        # FIXME: Doing it this way makes the update non-atomic if a method fails after an earlier one has succeeded.
-        methods = []
-        if params.name or params.description:
-            methods.append(self.quota_manager.rename_quota)
-        if params.amount:
-            methods.append(self.quota_manager.edit_quota)
-        if params.default == DefaultQuotaValues.NO:
-            methods.append(self.quota_manager.unset_quota_default)
-        elif params.default:
-            methods.append(self.quota_manager.set_quota_default)
-        if params.in_users or params.in_groups:
-            methods.append(self.quota_manager.manage_users_and_groups_for_quota)
-
-        messages = []
-        for method in methods:
-            message = method(quota, params)
-            messages.append(message)
-        return '; '.join(messages)
-
-    def delete(self, trans: ProvidesUserContext, id: EncodedDatabaseIdField, payload: Optional[DeleteQuotaPayload] = None) -> str:
-        """Marks a quota as deleted."""
-        quota = self.quota_manager.get_quota(trans, id, deleted=False)  # deleted quotas are not technically members of this collection
-        message = self.quota_manager.delete_quota(quota)
-        if payload and payload.purge:
-            message += self.quota_manager.purge_quota(quota)
-        return message
-
-    def undelete(self, trans: ProvidesUserContext, id: EncodedDatabaseIdField) -> str:
-        """Restores a previously deleted quota."""
-        quota = self.quota_manager.get_quota(trans, id, deleted=True)
-        return self.quota_manager.undelete_quota(quota)
-
-    def validate_in_users_and_groups(self, trans, payload):
-        """
-        For convenience, in_users and in_groups can be encoded IDs or emails/group names in the API.
-        """
-        def get_id(item, model_class, column):
-            try:
-                return trans.security.decode_id(item)
-            except Exception:
-                pass  # maybe an email/group name
-            # this will raise if the item is invalid
-            return trans.sa_session.query(model_class).filter(column == item).first().id
-        new_in_users = []
-        new_in_groups = []
-        invalid = []
-        for item in util.listify(payload.get('in_users', [])):
-            try:
-                new_in_users.append(get_id(item, model.User, model.User.email))
-            except Exception:
-                invalid.append(item)
-        for item in util.listify(payload.get('in_groups', [])):
-            try:
-                new_in_groups.append(get_id(item, model.Group, model.Group.name))
-            except Exception:
-                invalid.append(item)
-        if invalid:
-            msg = f"The following value(s) for associated users and/or groups could not be parsed: {', '.join(invalid)}."
-            msg += "  Valid values are email addresses of users, names of groups, or IDs of both."
-            raise Exception(msg)
-        payload['in_users'] = list(map(str, new_in_users))
-        payload['in_groups'] = list(map(str, new_in_groups))
-
-    def _url_for(self, *args, **kargs):
-        try:
-            return url_for(*args, **kargs)
-        except AttributeError:
-            return "*deprecated attribute not filled in by FastAPI server*"

lib/galaxy/managers/remote_files.py

@@ -9,8 +9,6 @@ from typing import (
     Optional,
 )
 
-from pydantic.tools import parse_obj_as
-
 from galaxy import exceptions
 from galaxy.app import MinimalManagerApp
 from galaxy.files import (
@@ -115,10 +113,11 @@ class RemoteFilesManager:
 
         return index
 
-    def get_files_source_plugins(self) -> FilesSourcePluginList:
+    def get_files_source_plugins(self, user_context: ProvidesUserContext) -> FilesSourcePluginList:
         """Display plugin information for each of the gxfiles:// URI targets available."""
-        plugins = self._file_sources.plugins_to_dict()
-        return parse_obj_as(FilesSourcePluginList, plugins)
+        user_file_source_context = ProvidesUserFileSourcesUserContext(user_context)
+        plugins = self._file_sources.plugins_to_dict(user_context=user_file_source_context)
+        return FilesSourcePluginList.parse_obj(plugins)
 
     @property
     def _file_sources(self) -> ConfiguredFileSources:

lib/galaxy/managers/sharable.py

@@ -11,23 +11,13 @@ A sharable Galaxy object:
 """
 import logging
 import re
-from enum import Enum
 from typing import (
-    List,
     Optional,
     Set,
-    Tuple,
     Type,
-    Union,
 )
 
-from pydantic import (
-    BaseModel,
-    Extra,
-    Field,
-)
 from sqlalchemy import (
-    false,
     true,
 )
 
@@ -44,124 +34,15 @@ from galaxy.model import (
     User,
     UserShareAssociation,
 )
-from galaxy.schema.fields import EncodedDatabaseIdField
+from galaxy.schema.schema import (
+    ShareWithExtra,
+    SharingOptions,
+)
 from galaxy.structured_app import MinimalManagerApp
 from galaxy.util import ready_name_for_url
 
 log = logging.getLogger(__name__)
 
-UserIdentifier = Union[EncodedDatabaseIdField, str]
-
-
-class SharingOptions(str, Enum):
-    """Options for sharing resources that may have restricted access to all or part of their contents."""
-    make_public = "make_public"
-    make_accessible_to_shared = "make_accessible_to_shared"
-    no_changes = "no_changes"
-
-
-class ShareWithExtra(BaseModel):
-    can_share: bool = Field(
-        False,
-        title="Can Share",
-        description="Indicates whether the resource can be directly shared or requires further actions.",
-    )
-
-    class Config:
-        extra = Extra.allow
-
-
-class ShareWithPayload(BaseModel):
-    user_ids: List[UserIdentifier] = Field(
-        ...,
-        title="User Identifiers",
-        description=(
-            "A collection of encoded IDs (or email addresses) of users "
-            "that this resource will be shared with."
-        ),
-    )
-    share_option: Optional[SharingOptions] = Field(
-        None,
-        title="Share Option",
-        description=(
-            "User choice for sharing resources which its contents may be restricted:\n"
-            " - None: The user did not choose anything yet or no option is needed.\n"
-            f" - {SharingOptions.make_public}: The contents of the resource will be made publicly accessible.\n"
-            f" - {SharingOptions.make_accessible_to_shared}: This will automatically create a new `sharing role` allowing protected contents to be accessed only by the desired users.\n"
-            f" - {SharingOptions.no_changes}: This won't change the current permissions for the contents. The user which this resource will be shared may not be able to access all its contents.\n"
-        ),
-    )
-
-
-class SetSlugPayload(BaseModel):
-    new_slug: str = Field(
-        ...,
-        title="New Slug",
-        description="The slug that will be used to access this shared item.",
-    )
-
-
-class UserEmail(BaseModel):
-    id: EncodedDatabaseIdField = Field(
-        ...,
-        title="User ID",
-        description="The encoded ID of the user.",
-    )
-    email: str = Field(
-        ...,
-        title="Email",
-        description="The email of the user.",
-    )
-
-
-class SharingStatus(BaseModel):
-    id: EncodedDatabaseIdField = Field(
-        ...,
-        title="ID",
-        description="The encoded ID of the resource to be shared.",
-    )
-    title: str = Field(
-        ...,
-        title="Title",
-        description="The title or name of the resource.",
-    )
-    importable: bool = Field(
-        ...,
-        title="Importable",
-        description="Whether this resource can be published using a link.",
-    )
-    published: bool = Field(
-        ...,
-        title="Published",
-        description="Whether this resource is currently published.",
-    )
-    users_shared_with: List[UserEmail] = Field(
-        [],
-        title="Users shared with",
-        description="The list of encoded ids for users the resource has been shared.",
-    )
-    username_and_slug: Optional[str] = Field(
-        None,
-        title="Username and slug",
-        description="The relative URL in the form of /u/{username}/{resource_single_char}/{slug}",
-    )
-
-
-class ShareWithStatus(SharingStatus):
-    errors: List[str] = Field(
-        [],
-        title="Errors",
-        description="Collection of messages indicating that the resource was not shared with some (or all users) due to an error.",
-    )
-    extra: Optional[ShareWithExtra] = Field(
-        None,
-        title="Extra",
-        description=(
-            "Optional extra information about this shareable resource that may be of interest. "
-            "The contents of this field depend on the particular resource."
-        ),
-    )
-
 
 class SharableModelManager(base.ModelManager, secured.OwnableManagerMixin, secured.AccessibleManagerMixin,
         taggable.TaggableManagerMixin, annotatable.AnnotatableManagerMixin, ratable.RatableManagerMixin):
@@ -653,108 +534,3 @@ class SlugBuilder:
         # Set slug and return.
         item.slug = new_slug
         return item.slug == cur_slug
-
-
-class ShareableService:
-    """ Provides the logic used by the API to share resources with other users."""
-
-    def __init__(self, manager: SharableModelManager, serializer: SharableModelSerializer) -> None:
-        self.manager = manager
-        self.serializer = serializer
-
-    def set_slug(self, trans, id: EncodedDatabaseIdField, payload: SetSlugPayload):
-        item = self._get_item_by_id(trans, id)
-        self.manager.set_slug(item, payload.new_slug, trans.user)
-
-    def sharing(self, trans, id: EncodedDatabaseIdField) -> SharingStatus:
-        """Gets the current sharing status of the item with the given id."""
-        item = self._get_item_by_id(trans, id)
-        return self._get_sharing_status(trans, item)
-
-    def enable_link_access(self, trans, id: EncodedDatabaseIdField) -> SharingStatus:
-        item = self._get_item_by_id(trans, id)
-        self.manager.make_importable(item)
-        return self._get_sharing_status(trans, item)
-
-    def disable_link_access(self, trans, id: EncodedDatabaseIdField) -> SharingStatus:
-        item = self._get_item_by_id(trans, id)
-        self.manager.make_non_importable(item)
-        return self._get_sharing_status(trans, item)
-
-    def publish(self, trans, id: EncodedDatabaseIdField) -> SharingStatus:
-        item = self._get_item_by_id(trans, id)
-        self.manager.publish(item)
-        return self._get_sharing_status(trans, item)
-
-    def unpublish(self, trans, id: EncodedDatabaseIdField) -> SharingStatus:
-        item = self._get_item_by_id(trans, id)
-        self.manager.unpublish(item)
-        return self._get_sharing_status(trans, item)
-
-    def share_with_users(self, trans, id: EncodedDatabaseIdField, payload: ShareWithPayload) -> ShareWithStatus:
-        item = self._get_item_by_id(trans, id)
-        users, errors = self._get_users(trans, payload.user_ids)
-        extra = self._share_with_options(trans, item, users, errors, payload.share_option)
-        base_status = self._get_sharing_status(trans, item)
-        status = ShareWithStatus.parse_obj(base_status)
-        status.extra = extra
-        status.errors.extend(errors)
-        return status
-
-    def _share_with_options(
-        self,
-        trans,
-        item,
-        users: Set[User],
-        errors: Set[str],
-        share_option: Optional[SharingOptions] = None,
-    ):
-        extra = self.manager.get_sharing_extra_information(trans, item, users, errors, share_option)
-        if not extra or extra.can_share:
-            self.manager.update_current_sharing_with_users(item, users)
-            extra = None
-        return extra
-
-    def _get_item_by_id(self, trans, id: EncodedDatabaseIdField):
-        class_name = self.manager.model_class.__name__
-        item = base.get_object(trans, id, class_name, check_ownership=True, check_accessible=True, deleted=False)
-        return item
-
-    def _get_sharing_status(self, trans, item):
-        status = self.serializer.serialize_to_view(item,
-            user=trans.user, trans=trans, default_view="sharing")
-        status["users_shared_with"] = [{"id": self.manager.app.security.encode_id(a.user.id), "email": a.user.email} for a in item.users_shared_with]
-        return SharingStatus.parse_obj(status)
-
-    def _get_users(self, trans, emails_or_ids: Optional[List] = None) -> Tuple[Set[User], Set[str]]:
-        if emails_or_ids is None:
-            raise exceptions.MessageException("Missing required user IDs or emails")
-        send_to_users: Set[User] = set()
-        send_to_err: Set[str] = set()
-        for email_or_id in set(emails_or_ids):
-            email_or_id = email_or_id.strip()
-            if not email_or_id:
-                continue
-
-            send_to_user = None
-            if '@' in email_or_id:
-                email_address = email_or_id
-                send_to_user = self.manager.user_manager.by_email(email_address,
-                    filters=[User.table.c.deleted == false()])
-            else:
-                try:
-                    decoded_user_id = trans.security.decode_id(email_or_id)
-                    send_to_user = self.manager.user_manager.by_id(decoded_user_id)
-                    if send_to_user.deleted:
-                        send_to_user = None
-                except exceptions.MalformedId:
-                    send_to_user = None
-
-            if not send_to_user:
-                send_to_err.add(f"{email_or_id} is not a valid Galaxy user.")
-            elif send_to_user == trans.user:
-                send_to_err.add("You cannot share resources with yourself.")
-            else:
-                send_to_users.add(send_to_user)
-
-        return send_to_users, send_to_err

lib/galaxy/managers/visualizations.py

@@ -71,19 +71,3 @@ class VisualizationDeserializer(sharable.SharableModelDeserializer):
         self.deserializers.update({
         })
         self.deserializable_keyset.update(self.deserializers.keys())
-
-
-class VisualizationsService:
-    """Common interface/service logic for interactions with visualizations in the context of the API.
-
-    Provides the logic of the actions invoked by API controllers and uses type definitions
-    and pydantic models to declare its parameters and return types.
-    """
-
-    def __init__(self, app: MinimalManagerApp, manager: VisualizationManager, serializer: VisualizationSerializer):
-        self.app = app
-        self.manager = manager
-        self.serializer = serializer
-        self.shareable_service = sharable.ShareableService(self.manager, self.serializer)
-
-    # TODO: add the rest of the API actions here and call them directly from the API controller

lib/galaxy/model/store/__init__.py

@@ -10,6 +10,7 @@ from json import (
     dumps,
     load,
 )
+from typing import Any, cast, Dict, List, Optional, Union
 from uuid import uuid4
 
 from bdbag import bdbag_api as bdb
@@ -27,6 +28,9 @@ from ..custom_types import json_encoder
 from ..item_attrs import add_item_annotation, get_item_annotation_str
 from ... import model
 
+
+ObjectKeyType = Union[str, int]
+
 ATTRS_FILENAME_HISTORY = 'history_attrs.txt'
 ATTRS_FILENAME_DATASETS = 'datasets_attrs.txt'
 ATTRS_FILENAME_JOBS = 'jobs_attrs.txt'
@@ -88,38 +92,43 @@ class ModelImportStore(metaclass=abc.ABCMeta):
         self.dataset_state_serialized = True
 
     @abc.abstractmethod
-    def defines_new_history(self):
+    def defines_new_history(self) -> bool:
         """Does this store define a new history to create."""
 
     @abc.abstractmethod
-    def new_history_properties(self):
+    def new_history_properties(self) -> Dict[str, Any]:
         """Dict of history properties if defines_new_history() is truthy."""
 
     @abc.abstractmethod
-    def datasets_properties(self):
+    def datasets_properties(self) -> List[Dict[str, Any]]:
         """Return a list of HDA properties."""
 
-    def library_properties(self):
+    def library_properties(self) -> List[Dict[str, Any]]:
         """Return a list of library properties."""
         return []
 
     @abc.abstractmethod
-    def collections_properties(self):
+    def collections_properties(self) -> List[Dict[str, Any]]:
         """Return a list of HDCA properties."""
 
     @abc.abstractmethod
-    def jobs_properties(self):
+    def jobs_properties(self) -> List[Dict[str, Any]]:
         """Return a list of jobs properties."""
 
     @abc.abstractproperty
-    def object_key(self):
+    def object_key(self) -> str:
         """Key used to connect objects in metadata.
 
         Legacy exports used 'hid' but associated objects may not be from the same history
         and a history may contain multiple objects with the same 'hid'.
         """
 
-    def trust_hid(self, obj_attrs):
+    @property
+    def file_source_root(self) -> Optional[str]:
+        """Source of valid file data."""
+        return None
+
+    def trust_hid(self, obj_attrs) -> bool:
         """Trust HID when importing objects into a new History."""
 
     @contextlib.contextmanager
@@ -182,6 +191,30 @@ class ModelImportStore(metaclass=abc.ABCMeta):
         self._import_implicit_collection_jobs(object_import_tracker)
         self._flush()
 
+    def _attach_dataset_hashes(self, dataset_or_file_attrs, dataset_instance):
+        if "hashes" in dataset_or_file_attrs:
+            for hash_attrs in dataset_or_file_attrs["hashes"]:
+                hash_obj = model.DatasetHash()
+                hash_obj.hash_value = hash_attrs["hash_value"]
+                hash_obj.hash_function = hash_attrs["hash_function"]
+                hash_obj.extra_files_path = hash_attrs["extra_files_path"]
+                dataset_instance.dataset.hashes.append(hash_obj)
+
+    def _attach_dataset_sources(self, dataset_or_file_attrs, dataset_instance):
+        if "sources" in dataset_or_file_attrs:
+            for source_attrs in dataset_or_file_attrs["sources"]:
+                source_obj = model.DatasetSource()
+                source_obj.source_uri = source_attrs["source_uri"]
+                source_obj.transform = source_attrs["transform"]
+                source_obj.extra_files_path = source_attrs["extra_files_path"]
+                for hash_attrs in source_attrs["hashes"]:
+                    hash_obj = model.DatasetSourceHash()
+                    hash_obj.hash_value = hash_attrs["hash_value"]
+                    hash_obj.hash_function = hash_attrs["hash_function"]
+                    source_obj.hashes.append(hash_obj)
+
+                dataset_instance.dataset.sources.append(source_obj)
+
     def _import_datasets(self, object_import_tracker, datasets_attrs, history, new_history, job):
         object_key = self.object_key
 
@@ -209,14 +242,9 @@ class ModelImportStore(metaclass=abc.ABCMeta):
                     for attribute in dataset_attributes:
                         if attribute in dataset_attrs["dataset"]:
                             setattr(dataset_instance.dataset, attribute, dataset_attrs["dataset"][attribute])
-                    if "hashes" in dataset_attrs["dataset"]:
-                        for hash_attrs in dataset_attrs["dataset"]["hashes"]:
-                            hash_obj = model.DatasetHash()
-                            hash_obj.hash_value = hash_attrs["hash_value"]
-                            hash_obj.hash_function = hash_attrs["hash_function"]
-                            hash_obj.extra_files_path = hash_attrs["extra_files_path"]
-                            dataset_instance.dataset.hashes.append(hash_obj)
-
+                    self._attach_dataset_hashes(dataset_attrs["dataset"], dataset_instance)
+                    # TODO: Once we have a test...
+                    #    self._attach_dataset_sources(dataset_attrs["dataset"], dataset_instance)
                     if 'id' in dataset_attrs["dataset"] and self.import_options.allow_edit:
                         dataset_instance.dataset.id = dataset_attrs["dataset"]['id']
 
@@ -252,6 +280,7 @@ class ModelImportStore(metaclass=abc.ABCMeta):
                 metadata = dataset_attrs['metadata']
 
                 model_class = dataset_attrs.get("model_class", "HistoryDatasetAssociation")
+                dataset_instance: model.DatasetInstance
                 if model_class == "HistoryDatasetAssociation":
                     # Create dataset and HDA.
                     dataset_instance = model.HistoryDatasetAssociation(name=dataset_attrs['name'],
@@ -298,29 +327,37 @@ class ModelImportStore(metaclass=abc.ABCMeta):
                 self._flush()
 
                 if model_class == "HistoryDatasetAssociation":
+                    hda = cast(model.HistoryDatasetAssociation, dataset_instance)
                     # don't use add_history to manage HID handling across full import to try to preserve
                     # HID structure.
-                    dataset_instance.history = history
+                    hda.history = history
                     if new_history and self.trust_hid(dataset_attrs):
-                        dataset_instance.hid = dataset_attrs['hid']
+                        hda.hid = dataset_attrs['hid']
                     else:
-                        object_import_tracker.requires_hid.append(dataset_instance)
+                        object_import_tracker.requires_hid.append(hda)
 
                 self._flush()
+                file_source_root = self.file_source_root
+
+                # If dataset is in the dictionary - we will assert this dataset is tied to the Galaxy instance
+                # and the import options are configured for allowing editing the dataset (e.g. for metadata setting).
+                # Otherwise, we will check for "file" information instead of dataset information - currently this includes
+                # "file_name", "extra_files_path".
                 if 'dataset' in dataset_attrs:
                     handle_dataset_object_edit(dataset_instance)
                 else:
                     file_name = dataset_attrs.get('file_name')
                     if file_name:
+                        assert file_source_root
                         # Do security check and move/copy dataset data.
-                        archive_path = os.path.abspath(os.path.join(self.archive_dir, file_name))
+                        archive_path = os.path.abspath(os.path.join(file_source_root, file_name))
                         if os.path.islink(archive_path):
                             raise MalformedContents(f"Invalid dataset path: {archive_path}")
 
                         temp_dataset_file_name = \
                             os.path.realpath(archive_path)
 
-                        if not in_directory(temp_dataset_file_name, self.archive_dir):
+                        if not in_directory(temp_dataset_file_name, file_source_root):
                             raise MalformedContents(f"Invalid dataset path: {temp_dataset_file_name}")
 
                     if not file_name or not os.path.exists(temp_dataset_file_name):
@@ -336,14 +373,15 @@ class ModelImportStore(metaclass=abc.ABCMeta):
                         # Import additional files if present. Histories exported previously might not have this attribute set.
                         dataset_extra_files_path = dataset_attrs.get('extra_files_path', None)
                         if dataset_extra_files_path:
+                            assert file_source_root
                             dir_name = dataset_instance.dataset.extra_files_path_name
-                            dataset_extra_files_path = os.path.join(self.archive_dir, dataset_extra_files_path)
+                            dataset_extra_files_path = os.path.join(file_source_root, dataset_extra_files_path)
                             for root, _dirs, files in safe_walk(dataset_extra_files_path):
                                 extra_dir = os.path.join(dir_name, root.replace(dataset_extra_files_path, '', 1).lstrip(os.path.sep))
                                 extra_dir = os.path.normpath(extra_dir)
                                 for extra_file in files:
                                     source = os.path.join(root, extra_file)
-                                    if not in_directory(source, self.archive_dir):
+                                    if not in_directory(source, file_source_root):
                                         raise MalformedContents(f"Invalid dataset path: {source}")
                                     self.object_store.update_from_file(
                                         dataset_instance.dataset, extra_dir=extra_dir,
@@ -353,6 +391,11 @@ class ModelImportStore(metaclass=abc.ABCMeta):
 
                     if dataset_instance.deleted:
                         dataset_instance.dataset.deleted = True
+                    file_metadata = dataset_attrs.get("file_metadata") or {}
+                    self._attach_dataset_hashes(file_metadata, dataset_instance)
+                    self._attach_dataset_sources(file_metadata, dataset_instance)
+                    if "created_from_basename" in file_metadata:
+                        dataset_instance.dataset.created_from_basename = file_metadata["created_from_basename"]
 
                 if model_class == "HistoryDatasetAssociation" and self.user:
                     add_item_annotation(self.sa_session, self.user, dataset_instance, dataset_attrs['annotation'])
@@ -623,27 +666,9 @@ class ModelImportStore(metaclass=abc.ABCMeta):
     def _import_jobs(self, object_import_tracker, history):
         object_key = self.object_key
 
-        def _find_hda(input_key):
-            hda = None
-            if input_key in object_import_tracker.hdas_by_key:
-                hda = object_import_tracker.hdas_by_key[input_key]
-            if input_key in object_import_tracker.hda_copied_from_sinks:
-                hda = object_import_tracker.hdas_by_key[object_import_tracker.hda_copied_from_sinks[input_key]]
-            return hda
-
-        def _find_hdca(input_key):
-            hdca = None
-            if input_key in object_import_tracker.hdcas_by_key:
-                hdca = object_import_tracker.hdcas_by_key[input_key]
-            if input_key in object_import_tracker.hdca_copied_from_sinks:
-                hdca = object_import_tracker.hdcas_by_key[object_import_tracker.hdca_copied_from_sinks[input_key]]
-            return hdca
-
-        def _find_dce(input_key):
-            dce = None
-            if input_key in object_import_tracker.dces_by_key:
-                dce = object_import_tracker.dces_by_key[input_key]
-            return dce
+        _find_hda = object_import_tracker.find_hda
+        _find_hdca = object_import_tracker.find_hdca
+        _find_dce = object_import_tracker.find_dce
 
         #
         # Create jobs.
@@ -758,6 +783,18 @@ class ObjectImportTracker:
 
     Needed to re-establish connections and such in multiple passes.
     """
+    libraries_by_key: Dict[ObjectKeyType, model.Library]
+    hdas_by_key: Dict[ObjectKeyType, model.HistoryDatasetAssociation]
+    hdas_by_id: Dict[int, model.HistoryDatasetAssociation]
+    hdcas_by_key: Dict[ObjectKeyType, model.HistoryDatasetCollectionAssociation]
+    hdcas_by_id: Dict[int, model.HistoryDatasetCollectionAssociation]
+    dces_by_key: Dict[ObjectKeyType, model.DatasetCollectionElement]
+    dces_by_id: Dict[int, model.DatasetCollectionElement]
+    lddas_by_key: Dict[ObjectKeyType, model.LibraryDatasetDatasetAssociation]
+    hda_copied_from_sinks: Dict[ObjectKeyType, ObjectKeyType]
+    hdca_copied_from_sinks: Dict[ObjectKeyType, ObjectKeyType]
+    jobs_by_key: Dict[ObjectKeyType, model.Job]
+    requires_hid: List[Union[model.HistoryDatasetAssociation, model.HistoryDatasetCollectionAssociation]]
 
     def __init__(self):
         self.libraries_by_key = {}
@@ -773,6 +810,28 @@ class ObjectImportTracker:
         self.jobs_by_key = {}
         self.requires_hid = []
 
+    def find_hda(self, input_key: ObjectKeyType) -> Optional[model.HistoryDatasetAssociation]:
+        hda = None
+        if input_key in self.hdas_by_key:
+            hda = self.hdas_by_key[input_key]
+        if input_key in self.hda_copied_from_sinks:
+            hda = self.hdas_by_key[self.hda_copied_from_sinks[input_key]]
+        return hda
+
+    def find_hdca(self, input_key: ObjectKeyType) -> Optional[model.HistoryDatasetCollectionAssociation]:
+        hdca = None
+        if input_key in self.hdcas_by_key:
+            hdca = self.hdcas_by_key[input_key]
+        if input_key in self.hdca_copied_from_sinks:
+            hdca = self.hdcas_by_key[self.hdca_copied_from_sinks[input_key]]
+        return hdca
+
+    def find_dce(self, input_key: ObjectKeyType) -> Optional[model.DatasetCollectionElement]:
+        dce = None
+        if input_key in self.dces_by_key:
+            dce = self.dces_by_key[input_key]
+        return dce
+
 
 def get_import_model_store_for_directory(archive_dir, **kwd):
     if not os.path.isdir(archive_dir):
@@ -785,6 +844,10 @@ def get_import_model_store_for_directory(archive_dir, **kwd):
 
 class BaseDirectoryImportModelStore(ModelImportStore):
 
+    @property
+    def file_source_root(self):
+        return self.archive_dir
+
     def defines_new_history(self):
         new_history_attributes = os.path.join(self.archive_dir, ATTRS_FILENAME_HISTORY)
         return os.path.exists(new_history_attributes)
@@ -1069,6 +1132,8 @@ class DirectoryModelExportStore(ModelExportStore):
                     shutil.copytree(src, dest)
                 else:
                     shutil.copyfile(src, dest)
+        else:
+            raise Exception(f"Unknown export_files parameter type encountered {self.export_files}")
 
         export_directory = self.export_directory
 

lib/galaxy/model/tags.py

@@ -156,6 +156,8 @@ class TagHandler:
 
     def apply_item_tag(self, user, item, name, value=None, flush=True):
         # Use lowercase name for searching/creating tag.
+        if name is None:
+            return
         lc_name = name.lower()
         # Get or create item-tag association.
         item_tag_assoc = self._get_item_tag_assoc(user, item, lc_name)

lib/galaxy/model/unittest_utils/__init__.py

@@ -0,0 +1,8 @@
+"""Interface provided by galaxy-data modules for unittest utilities for reuse by other modules."""
+from .data_app import GalaxyDataTestApp, GalaxyDataTestConfig
+
+
+__all__ = [
+    'GalaxyDataTestApp',
+    'GalaxyDataTestConfig',
+]

lib/galaxy/model/unittest_utils/data_app.py

@@ -0,0 +1,90 @@
+"""Mock GalaxyApp exposing config + functionality required for galaxy-data package.
+
+There is a more expansive MockApp in test/unit/unittest_utils - but it isn't packaged
+and it has dependencies from across the app. This mock application and config is
+more appropriate for testing galaxy-data functionality and will be included with
+galaxy-data.
+"""
+import os
+import shutil
+import tempfile
+
+from galaxy import model, objectstore
+from galaxy.datatypes import registry
+from galaxy.model.mapping import GalaxyModelMapping, init
+from galaxy.model.security import GalaxyRBACAgent
+from galaxy.security.idencoding import IdEncodingHelper
+from galaxy.util.bunch import Bunch
+
+
+GALAXY_TEST_UNITTEST_SECRET = '6e46ed6483a833c100e68cc3f1d0dd76'
+GALAXY_TEST_IN_MEMORY_DB_CONNECTION = "sqlite:///:memory:"
+
+
+class GalaxyDataTestConfig(Bunch):
+    """Minimal Galaxy mock config object that exposes and uses only what is needed for the galaxy-data package."""
+    security: IdEncodingHelper
+    database_connection: str
+    root: str
+    data_dir: str
+    _remove_root: bool
+
+    def __init__(self, root=None, **kwd):
+        Bunch.__init__(self, **kwd)
+        if not root:
+            root = tempfile.mkdtemp()
+            self._remove_root = True
+        else:
+            self._remove_root = False
+        self.root = root
+        self.data_dir = os.path.join(root, 'database')
+
+        self.security = IdEncodingHelper(id_secret=GALAXY_TEST_UNITTEST_SECRET)
+        self.database_connection = kwd.get('database_connection', GALAXY_TEST_IN_MEMORY_DB_CONNECTION)
+
+        # objectstore config values...
+        self.object_store_config_file = ''
+        self.object_store = 'disk'
+        self.object_store_check_old_style = False
+        self.object_store_cache_path = '/tmp/cache'
+        self.object_store_store_by = "uuid"
+
+        self.umask = os.umask(0o77)
+        self.gid = os.getgid()
+        # objectstore config directories...
+        self.jobs_directory = os.path.join(self.data_dir, 'jobs_directory')
+        self.new_file_path = os.path.join(self.data_dir, 'tmp')
+        self.file_path = os.path.join(self.data_dir, 'files')
+
+    def __del__(self):
+        if self._remove_root:
+            shutil.rmtree(self.root)
+
+
+class GalaxyDataTestApp():
+    """Minimal Galaxy mock app object that exposes and uses only what is needed for the galaxy-data package."""
+    security: IdEncodingHelper
+    model: GalaxyModelMapping
+    security_agent: GalaxyRBACAgent
+
+    def __init__(self, config: GalaxyDataTestConfig = None, **kwd):
+        config = config or GalaxyDataTestConfig(**kwd)
+        self.config = config
+        self.security = config.security
+        self.object_store = objectstore.build_object_store_from_config(self.config)
+        self.model = init("/tmp", self.config.database_connection, create_tables=True, object_store=self.object_store)
+        self.security_agent = self.model.security_agent
+        self.init_datatypes()
+
+    def init_datatypes(self):
+        datatypes_registry = registry.Registry()
+        datatypes_registry.load_datatypes()
+        model.set_datatypes_registry(datatypes_registry)
+        datatypes_registry.set_external_metadata_tool = MockSetExternalTool()
+        self.datatypes_registry = datatypes_registry
+
+
+class MockSetExternalTool:
+
+    def regenerate_imported_metadata_if_needed(self, *args, **kwds):
+        pass

lib/galaxy/objectstore/irods.py

@@ -5,7 +5,6 @@ import logging
 import os
 import shutil
 from datetime import datetime
-from functools import partial
 from pathlib import Path
 
 try:
@@ -347,30 +346,18 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin):
 
         collection_path = f"{self.home}/{str(subcollection_name)}"
         data_object_path = f"{collection_path}/{str(data_object_name)}"
-        data_obj = None
 
         try:
-            data_obj = self.session.data_objects.get(data_object_path)
+            cache_path = self._get_cache_path(rel_path)
+            self.session.data_objects.get(data_object_path, cache_path)
+            log.debug("Pulled data object '%s' into cache to %s", rel_path, cache_path)
+            return True
         except (DataObjectDoesNotExist, CollectionDoesNotExist):
             log.warning("Collection or data object (%s) does not exist", data_object_path)
             return False
         finally:
             log.debug("irods_pt _download: %s", ipt_timer)
 
-        if self.cache_size > 0 and data_obj.__sizeof__() > self.cache_size:
-            log.critical("File %s is larger (%s) than the cache size (%s). Cannot download.",
-                        rel_path, data_obj.__sizeof__(), self.cache_size)
-            log.debug("irods_pt _download: %s", ipt_timer)
-            return False
-
-        log.debug("Pulled data object '%s' into cache to %s", rel_path, self._get_cache_path(rel_path))
-
-        with data_obj.open('r') as data_obj_fp, open(self._get_cache_path(rel_path), "wb") as cache_fp:
-            for chunk in iter(partial(data_obj_fp.read, CHUNK_SIZE), b''):
-                cache_fp.write(chunk)
-        log.debug("irods_pt _download: %s", ipt_timer)
-        return True
-
     def _push_to_irods(self, rel_path, source_file=None, from_string=None):
         """
         Push the file pointed to by ``rel_path`` to the iRODS. Extract folder name
@@ -406,10 +393,10 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin):
             # Create sub-collection first
             self.session.collections.create(collection_path, recurse=True)
 
-            # Create data object
-            data_obj = self.session.data_objects.create(data_object_path, self.resource, **options)
-
             if from_string:
+                # Create data object
+                data_obj = self.session.data_objects.create(data_object_path, self.resource, **options)
+
                 # Save 'from_string' as a file
                 with data_obj.open('w') as data_obj_fp:
                     data_obj_fp.write(from_string)