From b6e2299c93d62681bd2f052de320cba13149c231 Mon Sep 17 00:00:00 2001
From: Helena Rasche <hxr@hx42.org>
Date: Wed, 3 Feb 2021 19:22:41 +0100
Subject: [PATCH 01/32] Persist scroll/url information, in both chrome &
 firefox

---
 config/plugins/webhooks/gtn/script.js | 65 ++++++++++++++++++++++++++-
 1 file changed, 63 insertions(+), 2 deletions(-)

diff --git a/config/plugins/webhooks/gtn/script.js b/config/plugins/webhooks/gtn/script.js
index 0d98fdaac03..0d395ab2052 100644
--- a/config/plugins/webhooks/gtn/script.js
+++ b/config/plugins/webhooks/gtn/script.js
@@ -1,4 +1,5 @@
 var gtnWebhookLoaded = false;
+var lastUpdate = 0;
 
 function removeOverlay() {
     document.getElementById("gtn-container").style.visibility = "hidden";
@@ -8,9 +9,44 @@ function showOverlay() {
     document.getElementById("gtn-container").style.visibility = "visible";
 }
 
+function getIframeUrl(){
+    var loc;
+    try {
+        loc = document.getElementById("gtn-embed").contentWindow.location.pathname;
+    } catch (e) {
+        loc = null;
+    }
+    return loc
+}
+
+function getIframeScroll() {
+    var loc;
+    try {
+        loc = parseInt(document.getElementById("gtn-embed").contentWindow.scrollY);
+    } catch (e) {
+        loc = 0;
+    }
+    return loc
+}
+
+function restoreLocation() {
+}
+
+function persistLocation() {
+    // Don't save every scroll event.
+    var time = new Date().getTime();
+    if ( time - lastUpdate < 1000 ) {
+        return;
+    }
+    lastUpdate = time;
+    window.localStorage.setItem('gtn-in-galaxy', `${getIframeScroll()} ${getIframeUrl()}`);
+}
+
 function addIframe() {
-    let url, message;
+    let url, message, onloadscroll;
     gtnWebhookLoaded = true;
+    let storedData = false;
+    let safe = false;
 
     // Test for the presence of /training-material/. If that is available we
     // can opt in the fancy click-to-run features. Otherwise we fallback to
@@ -24,7 +60,15 @@ function addIframe() {
                             <a href="https://docs.galaxyproject.org/en/master/admin/special_topics/gtn.html">Click to run</a> unavailable.
                         </span>`;
             } else {
-                url = "/training-material/";
+                safe = true;
+
+                var storedLocation = window.localStorage.getItem('gtn-in-galaxy');
+                if(storedLocation !== null && storedLocation.split(' ')[1] !== undefined && storedLocation.split(' ')[1].startsWith('/training-material/')) {
+                    onloadscroll = storedLocation.split(' ')[0];
+                    url = storedLocation.split(' ')[1];
+                } else {
+                    url = "/training-material/";
+                }
                 message = "";
             }
         })
@@ -48,8 +92,25 @@ function addIframe() {
                 removeOverlay();
             });
 
+            // Only setup the listener if it won't crash things.
+            if(safe) {
+                // Listen to the scroll position
+                document.getElementById("gtn-embed").contentWindow.addEventListener('scroll', () => {
+                    persistLocation();
+                });
+            }
+
             // Depends on the iframe being present
             document.getElementById("gtn-embed").addEventListener("load", () => {
+                // Save our current location when possible
+                if(onloadscroll !== undefined){
+                    document.getElementById('gtn-embed').contentWindow.scrollTo(0, parseInt(onloadscroll));
+                    onloadscroll = undefined;
+                }
+
+                if(safe) {
+                    persistLocation();
+                }
                 var gtn_tools = $("#gtn-embed").contents().find("span[data-tool]");
                 // Buttonify
                 gtn_tools.addClass("galaxy-proxy-active");

From c925772914c6ec3877e91c4fe7b44cb02ad962de Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Wed, 16 Dec 2020 08:07:00 +0100
Subject: [PATCH 02/32] Use nested state tracking with sqlalchemy-mutable

instead of the adapted gist recipe we've been using for a long time,
and which fails with:
```
Traceback (most recent call last):
  File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/workflow/run.py", line 190, in invoke
    incomplete_or_none = self._invoke_step(workflow_invocation_step)
  File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/workflow/run.py", line 266, in _invoke_step
    use_cached_job=self.workflow_invocation.use_cached_job)
  File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/workflow/modules.py", line 1761, in execute
    workflow_resource_parameters=resource_parameters
  File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/tools/execute.py", line 103, in execute
    execute_single_job(execution_slice, completed_jobs[i])
  File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/tools/execute.py", line 74, in execute_single_job
    execution_tracker.record_success(execution_slice, job, result)
  File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/tools/execute.py", line 432, in record_success
    self.job_callback(job)
  File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/workflow/modules.py", line 1759, in <lambda>
    job_callback=lambda job: self._handle_post_job_actions(step, job, invocation.replacement_dict),
  File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/workflow/modules.py", line 1812, in _handle_post_job_actions
    ActionBox.execute(self.trans.app, self.trans.sa_session, pja, job, replacement_dict)
  File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/jobs/actions/post.py", line 517, in execute
    ActionBox.actions[pja.action_type].execute(app, sa_session, pja, job, replacement_dict)
  File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/jobs/actions/post.py", line 103, in execute
    app.datatypes_registry.change_datatype(dataset_instance, action.action_arguments['newtype'])
  File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/datatypes/registry.py", line 590, in change_datatype
    data.init_meta(copy_from=data)
  File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/model/__init__.py", line 2783, in init_meta
    return self.datatype.init_meta(self, copy_from=copy_from)
  File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/datatypes/data.py", line 171, in init_meta
    dataset.metadata = copy_from.metadata
  File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/model/__init__.py", line 2705, in set_metadata
    self._metadata = self.metadata.make_dict_copy(bunch)
  File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/model/metadata.py", line 161, in make_dict_copy
    rval[key] = self.spec[key].param.make_copy(value, target_context=self, source_context=to_copy)
  File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/model/metadata.py", line 281, in make_copy
    return copy.deepcopy(value)
  File "/cvmfs/main.galaxyproject.org/deps/_conda/envs/_galaxy_/lib/python3.6/copy.py", line 161, in deepcopy
    y = copier(memo)
  File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/model/custom_types.py", line 232, in __deepcopy__
    return MutationList(MutationObj.coerce(self._key, copy.deepcopy(self[:])))
AttributeError: 'MutationList' object has no attribute '_key'
```
---
 .../dependencies/pipfiles/default/Pipfile     |   1 +
 .../pipfiles/default/pinned-requirements.txt  |   1 +
 lib/galaxy/model/custom_types.py              | 146 +-----------------
 3 files changed, 4 insertions(+), 144 deletions(-)

diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile
index 1e04dcf0216..9bbabc9700b 100644
--- a/lib/galaxy/dependencies/pipfiles/default/Pipfile
+++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile
@@ -77,6 +77,7 @@ kombu = "*"
 psutil = "*"
 pulsar-galaxy-lib = "==0.14.1"
 sqlalchemy-migrate = "*"
+sqlalchemy-mutable = "*"
 sqlitedict = "*"
 sqlparse = "*"
 svgwrite = "*"
diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
index a75bd2a035d..68de706b27a 100644
--- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
+++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
@@ -165,6 +165,7 @@ six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3
 social-auth-core[openidconnect]==3.3.0
 sortedcontainers==2.3.0
 sqlalchemy-migrate==0.13.0
+sqlalchemy-mutable==0.0.11
 sqlalchemy-utils==0.36.7
 sqlalchemy==1.3.20
 sqlitedict==1.7.0
diff --git a/lib/galaxy/model/custom_types.py b/lib/galaxy/model/custom_types.py
index 50840d38cbe..f70f8c906cb 100644
--- a/lib/galaxy/model/custom_types.py
+++ b/lib/galaxy/model/custom_types.py
@@ -9,13 +9,13 @@ from sys import getsizeof
 
 import numpy
 import sqlalchemy
-from sqlalchemy.ext.mutable import Mutable
 from sqlalchemy.types import (
     CHAR,
     LargeBinary,
     String,
     TypeDecorator
 )
+from sqlalchemy_mutable.mutable import Mutable
 
 from galaxy.util import (
     smart_str,
@@ -113,149 +113,7 @@ class JSONType(sqlalchemy.types.TypeDecorator):
         return (x == y)
 
 
-class MutationObj(Mutable):
-    """
-    Mutable JSONType for SQLAlchemy from original gist:
-    https://gist.github.com/dbarnett/1730610
-
-    Using minor changes from this fork of the gist:
-    https://gist.github.com/miracle2k/52a031cced285ba9b8cd
-
-    And other minor changes to make it work for us.
-    """
-    @classmethod
-    def coerce(cls, key, value):
-        if isinstance(value, dict) and not isinstance(value, MutationDict):
-            return MutationDict.coerce(key, value)
-        if isinstance(value, list) and not isinstance(value, MutationList):
-            return MutationList.coerce(key, value)
-        return value
-
-    @classmethod
-    def _listen_on_attribute(cls, attribute, coerce, parent_cls):
-        key = attribute.key
-        if parent_cls is not attribute.class_:
-            return
-
-        # rely on "propagate" here
-        parent_cls = attribute.class_
-
-        def load(state, *args):
-            val = state.dict.get(key, None)
-            if coerce and key not in state.unloaded:
-                val = cls.coerce(key, val)
-                state.dict[key] = val
-            if isinstance(val, cls):
-                val._parents[state.obj()] = key
-
-        def set(target, value, oldvalue, initiator):
-            if not isinstance(value, cls):
-                value = cls.coerce(key, value)
-            if isinstance(value, cls):
-                value._parents[target.obj()] = key
-            if isinstance(oldvalue, cls):
-                oldvalue._parents.pop(target.obj(), None)
-            return value
-
-        def pickle(state, state_dict):
-            val = state.dict.get(key, None)
-            if isinstance(val, cls):
-                if 'ext.mutable.values' not in state_dict:
-                    state_dict['ext.mutable.values'] = []
-                state_dict['ext.mutable.values'].append(val)
-
-        def unpickle(state, state_dict):
-            if 'ext.mutable.values' in state_dict:
-                for val in state_dict['ext.mutable.values']:
-                    val._parents[state.obj()] = key
-
-        sqlalchemy.event.listen(parent_cls, 'load', load, raw=True, propagate=True)
-        sqlalchemy.event.listen(parent_cls, 'refresh', load, raw=True, propagate=True)
-        sqlalchemy.event.listen(attribute, 'set', set, raw=True, retval=True, propagate=True)
-        sqlalchemy.event.listen(parent_cls, 'pickle', pickle, raw=True, propagate=True)
-        sqlalchemy.event.listen(parent_cls, 'unpickle', unpickle, raw=True, propagate=True)
-
-
-class MutationDict(MutationObj, dict):
-    @classmethod
-    def coerce(cls, key, value):
-        """Convert plain dictionary to MutationDict"""
-        self = MutationDict((k, MutationObj.coerce(key, v)) for (k, v) in value.items())
-        self._key = key
-        return self
-
-    def __setitem__(self, key, value):
-        if hasattr(self, '_key'):
-            value = MutationObj.coerce(self._key, value)
-        dict.__setitem__(self, key, value)
-        self.changed()
-
-    def __delitem__(self, key):
-        dict.__delitem__(self, key)
-        self.changed()
-
-    def __getstate__(self):
-        return dict(self)
-
-    def __setstate__(self, state):
-        self.update(state)
-
-
-class MutationList(MutationObj, list):
-    @classmethod
-    def coerce(cls, key, value):
-        """Convert plain list to MutationList"""
-        self = MutationList(MutationObj.coerce(key, v) for v in value)
-        self._key = key
-        return self
-
-    def __setitem__(self, idx, value):
-        list.__setitem__(self, idx, MutationObj.coerce(self._key, value))
-        self.changed()
-
-    def __setslice__(self, start, stop, values):
-        list.__setslice__(self, start, stop, (MutationObj.coerce(self._key, v) for v in values))
-        self.changed()
-
-    def __delitem__(self, idx):
-        list.__delitem__(self, idx)
-        self.changed()
-
-    def __delslice__(self, start, stop):
-        list.__delslice__(self, start, stop)
-        self.changed()
-
-    def __copy__(self):
-        return MutationList(MutationObj.coerce(self._key, self[:]))
-
-    def __deepcopy__(self, memo):
-        return MutationList(MutationObj.coerce(self._key, copy.deepcopy(self[:])))
-
-    def append(self, value):
-        list.append(self, MutationObj.coerce(self._key, value))
-        self.changed()
-
-    def insert(self, idx, value):
-        list.insert(self, idx, MutationObj.coerce(self._key, value))
-        self.changed()
-
-    def extend(self, values):
-        if hasattr(self, '_key'):
-            values = (MutationObj.coerce(self._key, value) for value in values)
-        list.extend(self, values)
-        self.changed()
-
-    def pop(self, *args, **kw):
-        value = list.pop(self, *args, **kw)
-        self.changed()
-        return value
-
-    def remove(self, value):
-        list.remove(self, value)
-        self.changed()
-
-
-MutationObj.associate_with(JSONType)
+Mutable.associate_with(JSONType)
 
 metadata_pickler = AliasPickleModule({
     ("cookbook.patterns", "Bunch"): ("galaxy.util.bunch", "Bunch")

From 05108eb931204f73216a50f1e80c88f641ff14db Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Wed, 3 Feb 2021 18:15:30 +0100
Subject: [PATCH 03/32] Work around bool not subclassable

---
 lib/galaxy/model/custom_types.py | 10 +++++++++-
 lib/galaxy/model/mapping.py      | 10 ++++++++--
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/model/custom_types.py b/lib/galaxy/model/custom_types.py
index f70f8c906cb..015544f801b 100644
--- a/lib/galaxy/model/custom_types.py
+++ b/lib/galaxy/model/custom_types.py
@@ -77,7 +77,7 @@ class GalaxyLargeBinary(LargeBinary):
         return process
 
 
-class JSONType(sqlalchemy.types.TypeDecorator):
+class BaseJSONType(sqlalchemy.types.TypeDecorator):
     """
     Represents an immutable structure as a json-encoded string.
 
@@ -113,6 +113,14 @@ class JSONType(sqlalchemy.types.TypeDecorator):
         return (x == y)
 
 
+class JSONType(BaseJSONType):
+    pass
+
+
+class SimpleJSONType(BaseJSONType):
+    pass
+
+
 Mutable.associate_with(JSONType)
 
 metadata_pickler = AliasPickleModule({
diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py
index db48b9b64c1..bfbff699551 100644
--- a/lib/galaxy/model/mapping.py
+++ b/lib/galaxy/model/mapping.py
@@ -38,7 +38,13 @@ from sqlalchemy.types import BigInteger
 
 from galaxy import model
 from galaxy.model.base import ModelMapping
-from galaxy.model.custom_types import JSONType, MetadataType, TrimmedString, UUIDType
+from galaxy.model.custom_types import (
+    JSONType,
+    MetadataType,
+    SimpleJSONType,
+    TrimmedString,
+    UUIDType,
+)
 from galaxy.model.orm.engine_factory import build_engine
 from galaxy.model.orm.now import now
 from galaxy.model.security import GalaxyRBACAgent
@@ -1123,7 +1129,7 @@ model.WorkflowInvocationStep.table = Table(
     Column("state", TrimmedString(64), index=True),
     Column("job_id", Integer, ForeignKey("job.id"), index=True, nullable=True),
     Column("implicit_collection_jobs_id", Integer, ForeignKey("implicit_collection_jobs.id"), index=True, nullable=True),
-    Column("action", JSONType, nullable=True))
+    Column("action", SimpleJSONType, nullable=True))
 
 model.WorkflowInvocationOutputDatasetAssociation.table = Table(
     "workflow_invocation_output_dataset_association", metadata,

From 47409c9a017f6e1593637f74c2f241fb5a766547 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Wed, 3 Feb 2021 18:39:04 +0100
Subject: [PATCH 04/32] Add sqlalchemy-mutable to data package

---
 packages/data/requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/data/requirements.txt b/packages/data/requirements.txt
index 291181dd209..1c1a0831a1d 100644
--- a/packages/data/requirements.txt
+++ b/packages/data/requirements.txt
@@ -12,5 +12,6 @@ pysam
 social-auth-core[openidconnect]==3.3.0
 SQLAlchemy
 sqlalchemy-migrate
+sqlalchemy-mutable
 sqlalchemy-utils
 WebOb

From c17b02ed8c3f4ac6adedb142589c40cbb8f8b847 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Thu, 4 Feb 2021 11:36:38 +0100
Subject: [PATCH 05/32] Switch to simpler sqlalchemy-json

sqlalchemy-mutable subclasses types, and not all libraries handle this
well (we would need to add CoercedBool isinstance check to the json
encoder), and pyyaml doesn't work with subclassed built-in types at all.
---
 .../dependencies/pipfiles/default/Pipfile     |  2 +-
 .../pipfiles/default/pinned-requirements.txt  |  2 +-
 lib/galaxy/model/custom_types.py              | 20 +++++++------------
 lib/galaxy/model/mapping.py                   | 16 +++++++--------
 .../migrate/versions/0001_initial_tables.py   |  2 +-
 .../versions/0003_security_and_libraries.py   |  6 +++---
 .../migrate/versions/0008_galaxy_forms.py     |  4 ++--
 .../versions/0019_request_library_folder.py   |  2 +-
 .../migrate/versions/0037_samples_library.py  |  4 ++--
 .../migrate/versions/0057_request_notify.py   |  2 +-
 .../versions/0067_populate_sequencer_table.py |  2 +-
 ..._add_tool_shed_repository_table_columns.py |  2 +-
 ...story_dataset_association_history_table.py |  2 +-
 .../migrate/versions/0149_dynamic_tools.py    |  2 +-
 packages/data/requirements.txt                |  2 +-
 15 files changed, 32 insertions(+), 38 deletions(-)

diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile
index 9bbabc9700b..36c4c6fc3fe 100644
--- a/lib/galaxy/dependencies/pipfiles/default/Pipfile
+++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile
@@ -76,8 +76,8 @@ boto = "*"
 kombu = "*"
 psutil = "*"
 pulsar-galaxy-lib = "==0.14.1"
+sqlalchemy-json = "*"
 sqlalchemy-migrate = "*"
-sqlalchemy-mutable = "*"
 sqlitedict = "*"
 sqlparse = "*"
 svgwrite = "*"
diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
index 68de706b27a..f596d8d628c 100644
--- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
+++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
@@ -164,8 +164,8 @@ simplejson==3.17.2; python_version >= '2.5' and python_version not in '3.0, 3.1,
 six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 social-auth-core[openidconnect]==3.3.0
 sortedcontainers==2.3.0
+sqlalchemy-json==0.4.0
 sqlalchemy-migrate==0.13.0
-sqlalchemy-mutable==0.0.11
 sqlalchemy-utils==0.36.7
 sqlalchemy==1.3.20
 sqlitedict==1.7.0
diff --git a/lib/galaxy/model/custom_types.py b/lib/galaxy/model/custom_types.py
index 015544f801b..238037d398c 100644
--- a/lib/galaxy/model/custom_types.py
+++ b/lib/galaxy/model/custom_types.py
@@ -15,7 +15,7 @@ from sqlalchemy.types import (
     String,
     TypeDecorator
 )
-from sqlalchemy_mutable.mutable import Mutable
+from sqlalchemy_json import mutable_json_type
 
 from galaxy.util import (
     smart_str,
@@ -77,7 +77,7 @@ class GalaxyLargeBinary(LargeBinary):
         return process
 
 
-class BaseJSONType(sqlalchemy.types.TypeDecorator):
+class SimpleJSONType(sqlalchemy.types.TypeDecorator):
     """
     Represents an immutable structure as a json-encoded string.
 
@@ -113,16 +113,6 @@ class BaseJSONType(sqlalchemy.types.TypeDecorator):
         return (x == y)
 
 
-class JSONType(BaseJSONType):
-    pass
-
-
-class SimpleJSONType(BaseJSONType):
-    pass
-
-
-Mutable.associate_with(JSONType)
-
 metadata_pickler = AliasPickleModule({
     ("cookbook.patterns", "Bunch"): ("galaxy.util.bunch", "Bunch")
 })
@@ -170,7 +160,7 @@ def total_size(o, handlers=None, verbose=False):
     return sizeof(o)
 
 
-class MetadataType(JSONType):
+class BaseMetadataType(SimpleJSONType):
     """
     Backward compatible metadata type. Can read pickles or JSON, but always
     writes in JSON.
@@ -203,6 +193,10 @@ class MetadataType(JSONType):
         return ret
 
 
+JSONType = mutable_json_type(dbtype=SimpleJSONType, nested=True)
+MetadataType = mutable_json_type(dbtype=BaseMetadataType, nested=True)
+
+
 class UUIDType(TypeDecorator):
     """
     Platform-independent UUID type.
diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py
index bfbff699551..04a65720ac9 100644
--- a/lib/galaxy/model/mapping.py
+++ b/lib/galaxy/model/mapping.py
@@ -193,7 +193,7 @@ model.DynamicTool.table = Table(
     Column("tool_directory", Unicode(255)),
     Column("hidden", Boolean, default=True),
     Column("active", Boolean, default=True),
-    Column("value", JSONType()),
+    Column("value", JSONType),
 )
 
 
@@ -239,7 +239,7 @@ model.HistoryDatasetAssociation.table = Table(
     Column("peek", TEXT, key="_peek"),
     Column("tool_version", TEXT),
     Column("extension", TrimmedString(64)),
-    Column("metadata", MetadataType(), key="_metadata"),
+    Column("metadata", MetadataType, key="_metadata"),
     Column("parent_id", Integer, ForeignKey("history_dataset_association.id"), nullable=True),
     Column("designation", TrimmedString(255)),
     Column("deleted", Boolean, index=True, default=False),
@@ -262,7 +262,7 @@ model.HistoryDatasetAssociationHistory.table = Table(
     Column("version", Integer),
     Column("name", TrimmedString(255)),
     Column("extension", TrimmedString(64)),
-    Column("metadata", MetadataType(), key="_metadata"),
+    Column("metadata", MetadataType, key="_metadata"),
     Column("extended_metadata_id", Integer, ForeignKey("extended_metadata.id"), index=True),
 )
 
@@ -517,7 +517,7 @@ model.LibraryDatasetDatasetAssociation.table = Table(
     Column("peek", TEXT, key="_peek"),
     Column("tool_version", TEXT),
     Column("extension", TrimmedString(64)),
-    Column("metadata", MetadataType(), key="_metadata"),
+    Column("metadata", MetadataType, key="_metadata"),
     Column("parent_id", Integer, ForeignKey("library_dataset_dataset_association.id"), nullable=True),
     Column("designation", TrimmedString(255)),
     Column("deleted", Boolean, index=True, default=False),
@@ -1054,7 +1054,7 @@ model.WorkflowRequestStepState.table = Table(
     Column("workflow_invocation_id", Integer,
         ForeignKey("workflow_invocation.id", onupdate="CASCADE", ondelete="CASCADE")),
     Column("workflow_step_id", Integer, ForeignKey("workflow_step.id")),
-    Column("value", JSONType))
+    Column("value", SimpleJSONType))
 
 model.WorkflowRequestInputParameter.table = Table(
     "workflow_request_input_parameters", metadata,
@@ -1225,9 +1225,9 @@ model.FormDefinition.table = Table(
     Column("name", TrimmedString(255), nullable=False),
     Column("desc", TEXT),
     Column("form_definition_current_id", Integer, ForeignKey("form_definition_current.id", use_alter=True), index=True, nullable=False),
-    Column("fields", JSONType()),
+    Column("fields", JSONType),
     Column("type", TrimmedString(255), index=True),
-    Column("layout", JSONType()))
+    Column("layout", JSONType))
 
 model.FormValues.table = Table(
     "form_values", metadata,
@@ -1235,7 +1235,7 @@ model.FormValues.table = Table(
     Column("create_time", DateTime, default=now),
     Column("update_time", DateTime, default=now, onupdate=now),
     Column("form_definition_id", Integer, ForeignKey("form_definition.id"), index=True),
-    Column("content", JSONType()))
+    Column("content", JSONType))
 
 model.Page.table = Table(
     "page", metadata,
diff --git a/lib/galaxy/model/migrate/versions/0001_initial_tables.py b/lib/galaxy/model/migrate/versions/0001_initial_tables.py
index 0b74124b505..6248b2ca0bb 100644
--- a/lib/galaxy/model/migrate/versions/0001_initial_tables.py
+++ b/lib/galaxy/model/migrate/versions/0001_initial_tables.py
@@ -58,7 +58,7 @@ HistoryDatasetAssociation_table = Table("history_dataset_association", metadata,
     Column("blurb", TrimmedString(255)),
     Column("peek", TEXT),
     Column("extension", TrimmedString(64)),
-    Column("metadata", MetadataType(), key="_metadata"),
+    Column("metadata", MetadataType, key="_metadata"),
     Column("parent_id", Integer, ForeignKey("history_dataset_association.id"), nullable=True),
     Column("designation", TrimmedString(255)),
     Column("deleted", Boolean, index=True, default=False),
diff --git a/lib/galaxy/model/migrate/versions/0003_security_and_libraries.py b/lib/galaxy/model/migrate/versions/0003_security_and_libraries.py
index 07711a97798..c79d72e94a8 100644
--- a/lib/galaxy/model/migrate/versions/0003_security_and_libraries.py
+++ b/lib/galaxy/model/migrate/versions/0003_security_and_libraries.py
@@ -170,7 +170,7 @@ LibraryDatasetDatasetAssociation_table = Table("library_dataset_dataset_associat
     Column("blurb", TrimmedString(255)),
     Column("peek", TEXT),
     Column("extension", TrimmedString(64)),
-    Column("metadata", MetadataType(), key="_metadata"),
+    Column("metadata", MetadataType, key="_metadata"),
     Column("parent_id", Integer, ForeignKey("library_dataset_dataset_association.id"), nullable=True),
     Column("designation", TrimmedString(255)),
     Column("deleted", Boolean, index=True, default=False),
@@ -211,7 +211,7 @@ LibraryItemInfoTemplateElement_table = Table("library_item_info_template_element
     Column("description", TEXT),
     Column("type", TEXT, default='string'),
     Column("order_id", Integer),
-    Column("options", JSONType()),
+    Column("options", JSONType),
     Column("library_item_info_template_id", Integer, ForeignKey("library_item_info_template.id")))
 Index("ix_liite_library_item_info_template_id", LibraryItemInfoTemplateElement_table.c.library_item_info_template_id)
 
@@ -262,7 +262,7 @@ LibraryItemInfoElement_table = Table("library_item_info_element", metadata,
     Column("id", Integer, primary_key=True),
     Column("create_time", DateTime, default=now),
     Column("update_time", DateTime, default=now, onupdate=now),
-    Column("contents", JSONType()),
+    Column("contents", JSONType),
     Column("library_item_info_id", Integer, ForeignKey("library_item_info.id"), index=True),
     Column("library_item_info_template_element_id", Integer, ForeignKey("library_item_info_template_element.id")))
 Index("ix_liie_library_item_info_template_element_id", LibraryItemInfoElement_table.c.library_item_info_template_element_id)
diff --git a/lib/galaxy/model/migrate/versions/0008_galaxy_forms.py b/lib/galaxy/model/migrate/versions/0008_galaxy_forms.py
index 2169901a1d8..8cd1b5d40d1 100644
--- a/lib/galaxy/model/migrate/versions/0008_galaxy_forms.py
+++ b/lib/galaxy/model/migrate/versions/0008_galaxy_forms.py
@@ -44,7 +44,7 @@ FormDefinition_table = Table('form_definition', metadata,
                              Column("name", TrimmedString(255), nullable=False),
                              Column("desc", TEXT),
                              Column("form_definition_current_id", Integer, ForeignKey("form_definition_current.id", use_alter=True), index=True, nullable=False),
-                             Column("fields", JSONType()))
+                             Column("fields", JSONType))
 
 FormDefinitionCurrent_table = Table('form_definition_current', metadata,
                                     Column("id", Integer, primary_key=True),
@@ -58,7 +58,7 @@ FormValues_table = Table('form_values', metadata,
                          Column("create_time", DateTime, default=now),
                          Column("update_time", DateTime, default=now, onupdate=now),
                          Column("form_definition_id", Integer, ForeignKey("form_definition.id"), index=True),
-                         Column("content", JSONType()))
+                         Column("content", JSONType))
 
 RequestType_table = Table('request_type', metadata,
                           Column("id", Integer, primary_key=True),
diff --git a/lib/galaxy/model/migrate/versions/0019_request_library_folder.py b/lib/galaxy/model/migrate/versions/0019_request_library_folder.py
index 5471ed09f55..17cfff7b1b1 100644
--- a/lib/galaxy/model/migrate/versions/0019_request_library_folder.py
+++ b/lib/galaxy/model/migrate/versions/0019_request_library_folder.py
@@ -38,7 +38,7 @@ def upgrade(migrate_engine):
     FormDefinition_table = Table("form_definition", metadata, autoload=True)
     col = Column("type", TrimmedString(255), index=True)
     add_column(col, FormDefinition_table, metadata, index_name='ix_form_definition_type')
-    col = Column("layout", JSONType())
+    col = Column("layout", JSONType)
     add_column(col, FormDefinition_table, metadata)
 
 
diff --git a/lib/galaxy/model/migrate/versions/0037_samples_library.py b/lib/galaxy/model/migrate/versions/0037_samples_library.py
index a51d9521bcd..154ec7c3839 100644
--- a/lib/galaxy/model/migrate/versions/0037_samples_library.py
+++ b/lib/galaxy/model/migrate/versions/0037_samples_library.py
@@ -35,7 +35,7 @@ def upgrade(migrate_engine):
     metadata.reflect()
 
     # Add the datatx_info column in 'request_type' table
-    col = Column("datatx_info", JSONType())
+    col = Column("datatx_info", JSONType)
     add_column(col, 'request_type', metadata)
 
     # Delete the library_id column in 'request' table
@@ -49,7 +49,7 @@ def upgrade(migrate_engine):
 
     # Add the dataset_files column in 'sample' table
     Sample_table = Table("sample", metadata, autoload=True)
-    col = Column("dataset_files", JSONType())
+    col = Column("dataset_files", JSONType)
     add_column(col, Sample_table, metadata)
 
     # Add the library_id column in 'sample' table
diff --git a/lib/galaxy/model/migrate/versions/0057_request_notify.py b/lib/galaxy/model/migrate/versions/0057_request_notify.py
index ab440bca442..f28c6562132 100644
--- a/lib/galaxy/model/migrate/versions/0057_request_notify.py
+++ b/lib/galaxy/model/migrate/versions/0057_request_notify.py
@@ -31,7 +31,7 @@ def upgrade(migrate_engine):
     Request_table = Table("request", metadata, autoload=True)
 
     # create the column again as JSONType
-    col = Column("notification", JSONType())
+    col = Column("notification", JSONType)
     add_column(col, Request_table, metadata)
 
     cmd = "SELECT id, user_id, notify FROM request"
diff --git a/lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py b/lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py
index cbd1280f99a..2af5e935635 100644
--- a/lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py
+++ b/lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py
@@ -221,7 +221,7 @@ def downgrade(migrate_engine):
 
     RequestType_table = Table("request_type", metadata, autoload=True)
     # create the 'datatx_info' column
-    col = Column("datatx_info", JSONType())
+    col = Column("datatx_info", JSONType)
     add_column(col, RequestType_table, metadata)
     # restore the datatx_info column data in the request_type table with data from
     # the sequencer and the form_values table
diff --git a/lib/galaxy/model/migrate/versions/0086_add_tool_shed_repository_table_columns.py b/lib/galaxy/model/migrate/versions/0086_add_tool_shed_repository_table_columns.py
index d44d1bb6409..0d8a6cb24ac 100644
--- a/lib/galaxy/model/migrate/versions/0086_add_tool_shed_repository_table_columns.py
+++ b/lib/galaxy/model/migrate/versions/0086_add_tool_shed_repository_table_columns.py
@@ -28,7 +28,7 @@ def upgrade(migrate_engine):
     metadata.reflect()
 
     ToolShedRepository_table = Table("tool_shed_repository", metadata, autoload=True)
-    c = Column("metadata", JSONType(), nullable=True)
+    c = Column("metadata", JSONType, nullable=True)
     add_column(c, ToolShedRepository_table, metadata)
     c = Column("includes_datatypes", Boolean, index=True, default=False)
     add_column(c, ToolShedRepository_table, metadata, index_name="ix_tool_shed_repository_includes_datatypes")
diff --git a/lib/galaxy/model/migrate/versions/0139_add_history_dataset_association_history_table.py b/lib/galaxy/model/migrate/versions/0139_add_history_dataset_association_history_table.py
index 99b221d7273..4e71c5d0ba9 100644
--- a/lib/galaxy/model/migrate/versions/0139_add_history_dataset_association_history_table.py
+++ b/lib/galaxy/model/migrate/versions/0139_add_history_dataset_association_history_table.py
@@ -35,7 +35,7 @@ HistoryDatasetAssociationHistory_table = Table(
     Column("version", Integer, index=True),
     Column("name", TrimmedString(255)),
     Column("extension", TrimmedString(64)),
-    Column("metadata", MetadataType(), key='_metadata'),
+    Column("metadata", MetadataType, key='_metadata'),
     Column("extended_metadata_id", Integer, ForeignKey("extended_metadata.id"), index=True),
 )
 
diff --git a/lib/galaxy/model/migrate/versions/0149_dynamic_tools.py b/lib/galaxy/model/migrate/versions/0149_dynamic_tools.py
index 504405dc27f..2374b2b37bf 100644
--- a/lib/galaxy/model/migrate/versions/0149_dynamic_tools.py
+++ b/lib/galaxy/model/migrate/versions/0149_dynamic_tools.py
@@ -27,7 +27,7 @@ DynamicTool_table = Table(
     Column("tool_directory", Unicode(255)),
     Column("hidden", Boolean),
     Column("active", Boolean),
-    Column("value", JSONType()),
+    Column("value", JSONType),
 )
 
 
diff --git a/packages/data/requirements.txt b/packages/data/requirements.txt
index 1c1a0831a1d..ea9af4eede7 100644
--- a/packages/data/requirements.txt
+++ b/packages/data/requirements.txt
@@ -11,7 +11,7 @@ pycryptodome
 pysam
 social-auth-core[openidconnect]==3.3.0
 SQLAlchemy
+sqlalchemy-json
 sqlalchemy-migrate
-sqlalchemy-mutable
 sqlalchemy-utils
 WebOb

From c9dc398bf0c07d75499b17eb97bcda4ea58f34f6 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Thu, 4 Feb 2021 13:29:36 +0100
Subject: [PATCH 06/32] Fix usage of email_alerts column

---
 lib/tool_shed/grids/repository_grids.py       |  3 +-
 lib/tool_shed/util/commit_util.py             |  4 +--
 lib/tool_shed/util/shed_util_common.py        |  2 +-
 .../webapp/controllers/repository.py          | 30 +++++--------------
 lib/tool_shed/webapp/model/mapping.py         |  2 +-
 5 files changed, 11 insertions(+), 30 deletions(-)

diff --git a/lib/tool_shed/grids/repository_grids.py b/lib/tool_shed/grids/repository_grids.py
index 9c5573d67fb..d60702f4a7f 100644
--- a/lib/tool_shed/grids/repository_grids.py
+++ b/lib/tool_shed/grids/repository_grids.py
@@ -1,4 +1,3 @@
-import json
 import logging
 
 from markupsafe import escape as escape_html
@@ -181,7 +180,7 @@ class RepositoryGrid(grids.Grid):
     class EmailAlertsColumn(grids.TextColumn):
 
         def get_value(self, trans, grid, repository):
-            if trans.user and repository.email_alerts and trans.user.email in json.loads(repository.email_alerts):
+            if trans.user and trans.user.email in repository.email_alerts:
                 return 'yes'
             return ''
 
diff --git a/lib/tool_shed/util/commit_util.py b/lib/tool_shed/util/commit_util.py
index 78fdc33c65c..2d5d511a3c7 100644
--- a/lib/tool_shed/util/commit_util.py
+++ b/lib/tool_shed/util/commit_util.py
@@ -1,6 +1,5 @@
 import bz2
 import gzip
-import json
 import logging
 import os
 import shutil
@@ -72,8 +71,7 @@ def check_file_contents_for_email_alerts(app):
     admin_users = app.config.get("admin_users", "").split(",")
     for repository in sa_session.query(app.model.Repository) \
                                 .filter(app.model.Repository.table.c.email_alerts != null()):
-        email_alerts = json.loads(repository.email_alerts)
-        for user_email in email_alerts:
+        for user_email in repository.email_alerts:
             if user_email in admin_users:
                 return True
     return False
diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py
index 747041ab96c..9f850de157e 100644
--- a/lib/tool_shed/util/shed_util_common.py
+++ b/lib/tool_shed/util/shed_util_common.py
@@ -385,7 +385,7 @@ def handle_email_alerts(app, host, repository, content_alert_str='', new_repo_al
                     email_alerts.append(user.email)
         else:
             subject = "Galaxy tool shed update alert for repository named %s" % str(repository.name)
-            email_alerts = json.loads(repository.email_alerts)
+            email_alerts = repository.email_alerts
         for email in email_alerts:
             to = email.strip()
             # Send it
diff --git a/lib/tool_shed/webapp/controllers/repository.py b/lib/tool_shed/webapp/controllers/repository.py
index 13ac6bf5987..0e363c77cd8 100644
--- a/lib/tool_shed/webapp/controllers/repository.py
+++ b/lib/tool_shed/webapp/controllers/repository.py
@@ -1658,10 +1658,6 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
         alerts = kwd.get('alerts', '')
         alerts_checked = CheckboxField.is_checked(alerts)
         category_ids = util.listify(kwd.get('category_id', ''))
-        if repository.email_alerts:
-            email_alerts = json.loads(repository.email_alerts)
-        else:
-            email_alerts = []
         allow_push = kwd.get('allow_push', '')
         error = False
         user = trans.user
@@ -1714,14 +1710,12 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
         elif kwd.get('receive_email_alerts_button', False):
             flush_needed = False
             if alerts_checked:
-                if user.email not in email_alerts:
-                    email_alerts.append(user.email)
-                    repository.email_alerts = json.dumps(email_alerts)
+                if user.email not in repository.email_alerts:
+                    repository.email_alerts.append(user.email)
                     flush_needed = True
             else:
-                if user.email in email_alerts:
-                    email_alerts.remove(user.email)
-                    repository.email_alerts = json.dumps(email_alerts)
+                if user.email in repository.email_alerts:
+                    repository.email_alerts.remove(user.email)
                     flush_needed = True
             if flush_needed:
                 trans.sa_session.add(repository)
@@ -1743,7 +1737,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
         for obj in options:
             label = obj.username
             allow_push_select_field.add_option(label, trans.security.encode_id(obj.id))
-        checked = alerts_checked or user.email in email_alerts
+        checked = alerts_checked or user.email in repository.email_alerts
         alerts_check_box = CheckboxField('alerts', value=checked)
         changeset_revision_select_field = grids_util.build_changeset_revision_select_field(trans,
                                                                                            repository,
@@ -2273,19 +2267,14 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
             flush_needed = False
             for repository_id in repository_ids:
                 repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id)
-                if repository.email_alerts:
-                    email_alerts = json.loads(repository.email_alerts)
-                else:
-                    email_alerts = []
+                email_alerts = repository.email_alerts
                 if user.email in email_alerts:
                     email_alerts.remove(user.email)
-                    repository.email_alerts = json.dumps(email_alerts)
                     trans.sa_session.add(repository)
                     flush_needed = True
                     total_alerts_removed += 1
                 else:
                     email_alerts.append(user.email)
-                    repository.email_alerts = json.dumps(email_alerts)
                     trans.sa_session.add(repository)
                     flush_needed = True
                     total_alerts_added += 1
@@ -2568,10 +2557,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
         display_reviews = kwd.get('display_reviews', False)
         alerts = kwd.get('alerts', '')
         alerts_checked = CheckboxField.is_checked(alerts)
-        if repository.email_alerts:
-            email_alerts = json.loads(repository.email_alerts)
-        else:
-            email_alerts = []
+        email_alerts = repository.email_alerts
         repository_dependencies = None
         user = trans.user
         if user and kwd.get('receive_email_alerts_button', False):
@@ -2579,12 +2565,10 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
             if alerts_checked:
                 if user.email not in email_alerts:
                     email_alerts.append(user.email)
-                    repository.email_alerts = json.dumps(email_alerts)
                     flush_needed = True
             else:
                 if user.email in email_alerts:
                     email_alerts.remove(user.email)
-                    repository.email_alerts = json.dumps(email_alerts)
                     flush_needed = True
             if flush_needed:
                 trans.sa_session.add(repository)
diff --git a/lib/tool_shed/webapp/model/mapping.py b/lib/tool_shed/webapp/model/mapping.py
index 332026540e7..271aded24f1 100644
--- a/lib/tool_shed/webapp/model/mapping.py
+++ b/lib/tool_shed/webapp/model/mapping.py
@@ -118,7 +118,7 @@ Repository.table = Table("repository", metadata,
                          Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True),
                          Column("private", Boolean, default=False),
                          Column("deleted", Boolean, index=True, default=False),
-                         Column("email_alerts", JSONType, nullable=True),
+                         Column("email_alerts", JSONType, nullable=True, default=list),
                          Column("times_downloaded", Integer),
                          Column("deprecated", Boolean, default=False))
 

From 43edcb48cd0814d6463967524acb8cfe719291a3 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Thu, 4 Feb 2021 17:48:21 +0100
Subject: [PATCH 07/32] Convert more parameters that take basic types to
 SimpleJSONType

---
 lib/galaxy/model/mapping.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py
index 04a65720ac9..280eec953b8 100644
--- a/lib/galaxy/model/mapping.py
+++ b/lib/galaxy/model/mapping.py
@@ -864,7 +864,7 @@ model.PostJobAction.table = Table(
     Column("workflow_step_id", Integer, ForeignKey("workflow_step.id"), index=True, nullable=True),
     Column("action_type", String(255), nullable=False),
     Column("output_name", String(255), nullable=True),
-    Column("action_arguments", JSONType, nullable=True))
+    Column("action_arguments", SimpleJSONType, nullable=True))
 
 model.PostJobActionAssociation.table = Table(
     "post_job_action_association", metadata,
@@ -1041,7 +1041,7 @@ model.WorkflowStepInput.table = Table(
     Column("scatter_type", TEXT),
     Column("value_from", JSONType),
     Column("value_from_type", TEXT),
-    Column("default_value", JSONType),
+    Column("default_value", SimpleJSONType),
     Column("default_value_set", Boolean, default=False),
     Column("runtime_value", Boolean, default=False),
     Index('ix_workflow_step_input_workflow_step_id_name_unique', "workflow_step_id", "name", unique=True, mysql_length={'name': 200}),
@@ -1070,7 +1070,7 @@ model.WorkflowRequestInputStepParameter.table = Table(
     Column("id", Integer, primary_key=True),
     Column("workflow_invocation_id", Integer, ForeignKey("workflow_invocation.id"), index=True),
     Column("workflow_step_id", Integer, ForeignKey("workflow_step.id")),
-    Column("parameter_value", JSONType),
+    Column("parameter_value", SimpleJSONType),
 )
 
 model.WorkflowRequestToInputDatasetAssociation.table = Table(

From 462861299a55f45bfd36a8a4520c8d45f45d7695 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Thu, 4 Feb 2021 20:25:27 +0100
Subject: [PATCH 08/32] Revert "Switch to simpler sqlalchemy-json"

This reverts commit fee1fd34cb9fae37c5cff4214f849b5af3c70283.
---
 .../dependencies/pipfiles/default/Pipfile     |  2 +-
 .../pipfiles/default/pinned-requirements.txt  |  2 +-
 lib/galaxy/model/custom_types.py              | 20 ++++++++++++-------
 lib/galaxy/model/mapping.py                   | 16 +++++++--------
 .../migrate/versions/0001_initial_tables.py   |  2 +-
 .../versions/0003_security_and_libraries.py   |  6 +++---
 .../migrate/versions/0008_galaxy_forms.py     |  4 ++--
 .../versions/0019_request_library_folder.py   |  2 +-
 .../migrate/versions/0037_samples_library.py  |  4 ++--
 .../migrate/versions/0057_request_notify.py   |  2 +-
 .../versions/0067_populate_sequencer_table.py |  2 +-
 ..._add_tool_shed_repository_table_columns.py |  2 +-
 ...story_dataset_association_history_table.py |  2 +-
 .../migrate/versions/0149_dynamic_tools.py    |  2 +-
 packages/data/requirements.txt                |  2 +-
 15 files changed, 38 insertions(+), 32 deletions(-)

diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile
index 36c4c6fc3fe..9bbabc9700b 100644
--- a/lib/galaxy/dependencies/pipfiles/default/Pipfile
+++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile
@@ -76,8 +76,8 @@ boto = "*"
 kombu = "*"
 psutil = "*"
 pulsar-galaxy-lib = "==0.14.1"
-sqlalchemy-json = "*"
 sqlalchemy-migrate = "*"
+sqlalchemy-mutable = "*"
 sqlitedict = "*"
 sqlparse = "*"
 svgwrite = "*"
diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
index f596d8d628c..68de706b27a 100644
--- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
+++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
@@ -164,8 +164,8 @@ simplejson==3.17.2; python_version >= '2.5' and python_version not in '3.0, 3.1,
 six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 social-auth-core[openidconnect]==3.3.0
 sortedcontainers==2.3.0
-sqlalchemy-json==0.4.0
 sqlalchemy-migrate==0.13.0
+sqlalchemy-mutable==0.0.11
 sqlalchemy-utils==0.36.7
 sqlalchemy==1.3.20
 sqlitedict==1.7.0
diff --git a/lib/galaxy/model/custom_types.py b/lib/galaxy/model/custom_types.py
index 238037d398c..015544f801b 100644
--- a/lib/galaxy/model/custom_types.py
+++ b/lib/galaxy/model/custom_types.py
@@ -15,7 +15,7 @@ from sqlalchemy.types import (
     String,
     TypeDecorator
 )
-from sqlalchemy_json import mutable_json_type
+from sqlalchemy_mutable.mutable import Mutable
 
 from galaxy.util import (
     smart_str,
@@ -77,7 +77,7 @@ class GalaxyLargeBinary(LargeBinary):
         return process
 
 
-class SimpleJSONType(sqlalchemy.types.TypeDecorator):
+class BaseJSONType(sqlalchemy.types.TypeDecorator):
     """
     Represents an immutable structure as a json-encoded string.
 
@@ -113,6 +113,16 @@ class SimpleJSONType(sqlalchemy.types.TypeDecorator):
         return (x == y)
 
 
+class JSONType(BaseJSONType):
+    pass
+
+
+class SimpleJSONType(BaseJSONType):
+    pass
+
+
+Mutable.associate_with(JSONType)
+
 metadata_pickler = AliasPickleModule({
     ("cookbook.patterns", "Bunch"): ("galaxy.util.bunch", "Bunch")
 })
@@ -160,7 +170,7 @@ def total_size(o, handlers=None, verbose=False):
     return sizeof(o)
 
 
-class BaseMetadataType(SimpleJSONType):
+class MetadataType(JSONType):
     """
     Backward compatible metadata type. Can read pickles or JSON, but always
     writes in JSON.
@@ -193,10 +203,6 @@ class BaseMetadataType(SimpleJSONType):
         return ret
 
 
-JSONType = mutable_json_type(dbtype=SimpleJSONType, nested=True)
-MetadataType = mutable_json_type(dbtype=BaseMetadataType, nested=True)
-
-
 class UUIDType(TypeDecorator):
     """
     Platform-independent UUID type.
diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py
index 280eec953b8..90ee80a21d8 100644
--- a/lib/galaxy/model/mapping.py
+++ b/lib/galaxy/model/mapping.py
@@ -193,7 +193,7 @@ model.DynamicTool.table = Table(
     Column("tool_directory", Unicode(255)),
     Column("hidden", Boolean, default=True),
     Column("active", Boolean, default=True),
-    Column("value", JSONType),
+    Column("value", JSONType()),
 )
 
 
@@ -239,7 +239,7 @@ model.HistoryDatasetAssociation.table = Table(
     Column("peek", TEXT, key="_peek"),
     Column("tool_version", TEXT),
     Column("extension", TrimmedString(64)),
-    Column("metadata", MetadataType, key="_metadata"),
+    Column("metadata", MetadataType(), key="_metadata"),
     Column("parent_id", Integer, ForeignKey("history_dataset_association.id"), nullable=True),
     Column("designation", TrimmedString(255)),
     Column("deleted", Boolean, index=True, default=False),
@@ -262,7 +262,7 @@ model.HistoryDatasetAssociationHistory.table = Table(
     Column("version", Integer),
     Column("name", TrimmedString(255)),
     Column("extension", TrimmedString(64)),
-    Column("metadata", MetadataType, key="_metadata"),
+    Column("metadata", MetadataType(), key="_metadata"),
     Column("extended_metadata_id", Integer, ForeignKey("extended_metadata.id"), index=True),
 )
 
@@ -517,7 +517,7 @@ model.LibraryDatasetDatasetAssociation.table = Table(
     Column("peek", TEXT, key="_peek"),
     Column("tool_version", TEXT),
     Column("extension", TrimmedString(64)),
-    Column("metadata", MetadataType, key="_metadata"),
+    Column("metadata", MetadataType(), key="_metadata"),
     Column("parent_id", Integer, ForeignKey("library_dataset_dataset_association.id"), nullable=True),
     Column("designation", TrimmedString(255)),
     Column("deleted", Boolean, index=True, default=False),
@@ -1054,7 +1054,7 @@ model.WorkflowRequestStepState.table = Table(
     Column("workflow_invocation_id", Integer,
         ForeignKey("workflow_invocation.id", onupdate="CASCADE", ondelete="CASCADE")),
     Column("workflow_step_id", Integer, ForeignKey("workflow_step.id")),
-    Column("value", SimpleJSONType))
+    Column("value", JSONType))
 
 model.WorkflowRequestInputParameter.table = Table(
     "workflow_request_input_parameters", metadata,
@@ -1225,9 +1225,9 @@ model.FormDefinition.table = Table(
     Column("name", TrimmedString(255), nullable=False),
     Column("desc", TEXT),
     Column("form_definition_current_id", Integer, ForeignKey("form_definition_current.id", use_alter=True), index=True, nullable=False),
-    Column("fields", JSONType),
+    Column("fields", JSONType()),
     Column("type", TrimmedString(255), index=True),
-    Column("layout", JSONType))
+    Column("layout", JSONType()))
 
 model.FormValues.table = Table(
     "form_values", metadata,
@@ -1235,7 +1235,7 @@ model.FormValues.table = Table(
     Column("create_time", DateTime, default=now),
     Column("update_time", DateTime, default=now, onupdate=now),
     Column("form_definition_id", Integer, ForeignKey("form_definition.id"), index=True),
-    Column("content", JSONType))
+    Column("content", JSONType()))
 
 model.Page.table = Table(
     "page", metadata,
diff --git a/lib/galaxy/model/migrate/versions/0001_initial_tables.py b/lib/galaxy/model/migrate/versions/0001_initial_tables.py
index 6248b2ca0bb..0b74124b505 100644
--- a/lib/galaxy/model/migrate/versions/0001_initial_tables.py
+++ b/lib/galaxy/model/migrate/versions/0001_initial_tables.py
@@ -58,7 +58,7 @@ HistoryDatasetAssociation_table = Table("history_dataset_association", metadata,
     Column("blurb", TrimmedString(255)),
     Column("peek", TEXT),
     Column("extension", TrimmedString(64)),
-    Column("metadata", MetadataType, key="_metadata"),
+    Column("metadata", MetadataType(), key="_metadata"),
     Column("parent_id", Integer, ForeignKey("history_dataset_association.id"), nullable=True),
     Column("designation", TrimmedString(255)),
     Column("deleted", Boolean, index=True, default=False),
diff --git a/lib/galaxy/model/migrate/versions/0003_security_and_libraries.py b/lib/galaxy/model/migrate/versions/0003_security_and_libraries.py
index c79d72e94a8..07711a97798 100644
--- a/lib/galaxy/model/migrate/versions/0003_security_and_libraries.py
+++ b/lib/galaxy/model/migrate/versions/0003_security_and_libraries.py
@@ -170,7 +170,7 @@ LibraryDatasetDatasetAssociation_table = Table("library_dataset_dataset_associat
     Column("blurb", TrimmedString(255)),
     Column("peek", TEXT),
     Column("extension", TrimmedString(64)),
-    Column("metadata", MetadataType, key="_metadata"),
+    Column("metadata", MetadataType(), key="_metadata"),
     Column("parent_id", Integer, ForeignKey("library_dataset_dataset_association.id"), nullable=True),
     Column("designation", TrimmedString(255)),
     Column("deleted", Boolean, index=True, default=False),
@@ -211,7 +211,7 @@ LibraryItemInfoTemplateElement_table = Table("library_item_info_template_element
     Column("description", TEXT),
     Column("type", TEXT, default='string'),
     Column("order_id", Integer),
-    Column("options", JSONType),
+    Column("options", JSONType()),
     Column("library_item_info_template_id", Integer, ForeignKey("library_item_info_template.id")))
 Index("ix_liite_library_item_info_template_id", LibraryItemInfoTemplateElement_table.c.library_item_info_template_id)
 
@@ -262,7 +262,7 @@ LibraryItemInfoElement_table = Table("library_item_info_element", metadata,
     Column("id", Integer, primary_key=True),
     Column("create_time", DateTime, default=now),
     Column("update_time", DateTime, default=now, onupdate=now),
-    Column("contents", JSONType),
+    Column("contents", JSONType()),
     Column("library_item_info_id", Integer, ForeignKey("library_item_info.id"), index=True),
     Column("library_item_info_template_element_id", Integer, ForeignKey("library_item_info_template_element.id")))
 Index("ix_liie_library_item_info_template_element_id", LibraryItemInfoElement_table.c.library_item_info_template_element_id)
diff --git a/lib/galaxy/model/migrate/versions/0008_galaxy_forms.py b/lib/galaxy/model/migrate/versions/0008_galaxy_forms.py
index 8cd1b5d40d1..2169901a1d8 100644
--- a/lib/galaxy/model/migrate/versions/0008_galaxy_forms.py
+++ b/lib/galaxy/model/migrate/versions/0008_galaxy_forms.py
@@ -44,7 +44,7 @@ FormDefinition_table = Table('form_definition', metadata,
                              Column("name", TrimmedString(255), nullable=False),
                              Column("desc", TEXT),
                              Column("form_definition_current_id", Integer, ForeignKey("form_definition_current.id", use_alter=True), index=True, nullable=False),
-                             Column("fields", JSONType))
+                             Column("fields", JSONType()))
 
 FormDefinitionCurrent_table = Table('form_definition_current', metadata,
                                     Column("id", Integer, primary_key=True),
@@ -58,7 +58,7 @@ FormValues_table = Table('form_values', metadata,
                          Column("create_time", DateTime, default=now),
                          Column("update_time", DateTime, default=now, onupdate=now),
                          Column("form_definition_id", Integer, ForeignKey("form_definition.id"), index=True),
-                         Column("content", JSONType))
+                         Column("content", JSONType()))
 
 RequestType_table = Table('request_type', metadata,
                           Column("id", Integer, primary_key=True),
diff --git a/lib/galaxy/model/migrate/versions/0019_request_library_folder.py b/lib/galaxy/model/migrate/versions/0019_request_library_folder.py
index 17cfff7b1b1..5471ed09f55 100644
--- a/lib/galaxy/model/migrate/versions/0019_request_library_folder.py
+++ b/lib/galaxy/model/migrate/versions/0019_request_library_folder.py
@@ -38,7 +38,7 @@ def upgrade(migrate_engine):
     FormDefinition_table = Table("form_definition", metadata, autoload=True)
     col = Column("type", TrimmedString(255), index=True)
     add_column(col, FormDefinition_table, metadata, index_name='ix_form_definition_type')
-    col = Column("layout", JSONType)
+    col = Column("layout", JSONType())
     add_column(col, FormDefinition_table, metadata)
 
 
diff --git a/lib/galaxy/model/migrate/versions/0037_samples_library.py b/lib/galaxy/model/migrate/versions/0037_samples_library.py
index 154ec7c3839..a51d9521bcd 100644
--- a/lib/galaxy/model/migrate/versions/0037_samples_library.py
+++ b/lib/galaxy/model/migrate/versions/0037_samples_library.py
@@ -35,7 +35,7 @@ def upgrade(migrate_engine):
     metadata.reflect()
 
     # Add the datatx_info column in 'request_type' table
-    col = Column("datatx_info", JSONType)
+    col = Column("datatx_info", JSONType())
     add_column(col, 'request_type', metadata)
 
     # Delete the library_id column in 'request' table
@@ -49,7 +49,7 @@ def upgrade(migrate_engine):
 
     # Add the dataset_files column in 'sample' table
     Sample_table = Table("sample", metadata, autoload=True)
-    col = Column("dataset_files", JSONType)
+    col = Column("dataset_files", JSONType())
     add_column(col, Sample_table, metadata)
 
     # Add the library_id column in 'sample' table
diff --git a/lib/galaxy/model/migrate/versions/0057_request_notify.py b/lib/galaxy/model/migrate/versions/0057_request_notify.py
index f28c6562132..ab440bca442 100644
--- a/lib/galaxy/model/migrate/versions/0057_request_notify.py
+++ b/lib/galaxy/model/migrate/versions/0057_request_notify.py
@@ -31,7 +31,7 @@ def upgrade(migrate_engine):
     Request_table = Table("request", metadata, autoload=True)
 
     # create the column again as JSONType
-    col = Column("notification", JSONType)
+    col = Column("notification", JSONType())
     add_column(col, Request_table, metadata)
 
     cmd = "SELECT id, user_id, notify FROM request"
diff --git a/lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py b/lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py
index 2af5e935635..cbd1280f99a 100644
--- a/lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py
+++ b/lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py
@@ -221,7 +221,7 @@ def downgrade(migrate_engine):
 
     RequestType_table = Table("request_type", metadata, autoload=True)
     # create the 'datatx_info' column
-    col = Column("datatx_info", JSONType)
+    col = Column("datatx_info", JSONType())
     add_column(col, RequestType_table, metadata)
     # restore the datatx_info column data in the request_type table with data from
     # the sequencer and the form_values table
diff --git a/lib/galaxy/model/migrate/versions/0086_add_tool_shed_repository_table_columns.py b/lib/galaxy/model/migrate/versions/0086_add_tool_shed_repository_table_columns.py
index 0d8a6cb24ac..d44d1bb6409 100644
--- a/lib/galaxy/model/migrate/versions/0086_add_tool_shed_repository_table_columns.py
+++ b/lib/galaxy/model/migrate/versions/0086_add_tool_shed_repository_table_columns.py
@@ -28,7 +28,7 @@ def upgrade(migrate_engine):
     metadata.reflect()
 
     ToolShedRepository_table = Table("tool_shed_repository", metadata, autoload=True)
-    c = Column("metadata", JSONType, nullable=True)
+    c = Column("metadata", JSONType(), nullable=True)
     add_column(c, ToolShedRepository_table, metadata)
     c = Column("includes_datatypes", Boolean, index=True, default=False)
     add_column(c, ToolShedRepository_table, metadata, index_name="ix_tool_shed_repository_includes_datatypes")
diff --git a/lib/galaxy/model/migrate/versions/0139_add_history_dataset_association_history_table.py b/lib/galaxy/model/migrate/versions/0139_add_history_dataset_association_history_table.py
index 4e71c5d0ba9..99b221d7273 100644
--- a/lib/galaxy/model/migrate/versions/0139_add_history_dataset_association_history_table.py
+++ b/lib/galaxy/model/migrate/versions/0139_add_history_dataset_association_history_table.py
@@ -35,7 +35,7 @@ HistoryDatasetAssociationHistory_table = Table(
     Column("version", Integer, index=True),
     Column("name", TrimmedString(255)),
     Column("extension", TrimmedString(64)),
-    Column("metadata", MetadataType, key='_metadata'),
+    Column("metadata", MetadataType(), key='_metadata'),
     Column("extended_metadata_id", Integer, ForeignKey("extended_metadata.id"), index=True),
 )
 
diff --git a/lib/galaxy/model/migrate/versions/0149_dynamic_tools.py b/lib/galaxy/model/migrate/versions/0149_dynamic_tools.py
index 2374b2b37bf..504405dc27f 100644
--- a/lib/galaxy/model/migrate/versions/0149_dynamic_tools.py
+++ b/lib/galaxy/model/migrate/versions/0149_dynamic_tools.py
@@ -27,7 +27,7 @@ DynamicTool_table = Table(
     Column("tool_directory", Unicode(255)),
     Column("hidden", Boolean),
     Column("active", Boolean),
-    Column("value", JSONType),
+    Column("value", JSONType()),
 )
 
 
diff --git a/packages/data/requirements.txt b/packages/data/requirements.txt
index ea9af4eede7..1c1a0831a1d 100644
--- a/packages/data/requirements.txt
+++ b/packages/data/requirements.txt
@@ -11,7 +11,7 @@ pycryptodome
 pysam
 social-auth-core[openidconnect]==3.3.0
 SQLAlchemy
-sqlalchemy-json
 sqlalchemy-migrate
+sqlalchemy-mutable
 sqlalchemy-utils
 WebOb

From d2b74ca94c496e7fba16afd367d7e45caf8a25be Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Thu, 4 Feb 2021 21:20:38 +0100
Subject: [PATCH 09/32] Add class docstrings

---
 lib/galaxy/model/custom_types.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/model/custom_types.py b/lib/galaxy/model/custom_types.py
index 015544f801b..48acb69047a 100644
--- a/lib/galaxy/model/custom_types.py
+++ b/lib/galaxy/model/custom_types.py
@@ -113,11 +113,13 @@ class BaseJSONType(sqlalchemy.types.TypeDecorator):
         return (x == y)
 
 
-class JSONType(BaseJSONType):
+class SimpleJSONType(BaseJSONType):
+    """SQLAlchemy column type that does not track mutations to mutable data."""
     pass
 
 
-class SimpleJSONType(BaseJSONType):
+class JSONType(BaseJSONType):
+    """SQLAlchemy column type that tracks mutations to mutable data."""
     pass
 
 

From f409ec6b2156ae4bba54fb3bbebe8ab63efddef7 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 5 Feb 2021 09:59:22 +0100
Subject: [PATCH 10/32] Fix gxformat2 roundtripping

---
 lib/galaxy/model/custom_types.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/galaxy/model/custom_types.py b/lib/galaxy/model/custom_types.py
index 48acb69047a..f77c80c6274 100644
--- a/lib/galaxy/model/custom_types.py
+++ b/lib/galaxy/model/custom_types.py
@@ -16,6 +16,8 @@ from sqlalchemy.types import (
     TypeDecorator
 )
 from sqlalchemy_mutable.mutable import Mutable
+# For compatibility with custom yaml dumping in gxformat2
+from sqlalchemy_mutable.mutable_dict import MutableDict as MutationDict  # noqa: F401
 
 from galaxy.util import (
     smart_str,

From 2c50b903b9878058519e882410373f38c6935138 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 5 Feb 2021 11:01:59 +0100
Subject: [PATCH 11/32] Eliminate unncessary use of OrderedDict

Dictionary sort order is stable since Python 3.6, which is the oldest version we
support.
---
 lib/galaxy/config/config_manage.py            | 19 ++++------
 lib/galaxy/datatypes/binary.py                |  3 +-
 lib/galaxy/datatypes/data.py                  | 11 +++---
 .../display_applications/application.py       |  9 ++---
 lib/galaxy/datatypes/registry.py              |  9 ++---
 lib/galaxy/datatypes/util/gff_util.py         |  3 +-
 lib/galaxy/job_execution/output_collect.py    |  7 ++--
 lib/galaxy/managers/collections.py            | 11 +++---
 lib/galaxy/managers/executables.py            |  4 +-
 lib/galaxy/managers/markdown_util.py          |  5 +--
 .../model/dataset_collections/builder.py      |  8 ++--
 lib/galaxy/model/store/discover.py            |  3 +-
 lib/galaxy/objectstore/__init__.py            |  3 +-
 lib/galaxy/openid/providers.py                |  5 +--
 .../tool_shed/galaxy_install/migrate/check.py |  5 +--
 .../galaxy_install/tool_migration_manager.py  |  5 +--
 lib/galaxy/tool_shed/tool_shed_registry.py    |  9 ++---
 lib/galaxy/tool_util/cwl/parser.py            |  3 +-
 lib/galaxy/tool_util/cwl/representation.py    |  3 +-
 lib/galaxy/tool_util/deps/__init__.py         |  5 +--
 lib/galaxy/tool_util/parser/cwl.py            |  5 +--
 lib/galaxy/tool_util/parser/output_objects.py |  4 +-
 lib/galaxy/tool_util/parser/xml.py            |  7 ++--
 lib/galaxy/tool_util/parser/yaml.py           |  6 +--
 lib/galaxy/tool_util/verify/interactor.py     | 12 ++----
 lib/galaxy/tools/__init__.py                  | 37 +++++++++----------
 lib/galaxy/tools/actions/__init__.py          |  7 ++--
 lib/galaxy/tools/actions/history_imp_exp.py   |  5 +--
 lib/galaxy/tools/actions/metadata.py          |  3 +-
 lib/galaxy/tools/actions/model_operations.py  |  3 +-
 lib/galaxy/tools/actions/upload_common.py     |  3 +-
 lib/galaxy/tools/data/__init__.py             |  3 +-
 lib/galaxy/tools/data_manager/manager.py      |  9 ++---
 lib/galaxy/tools/execute.py                   |  2 +-
 lib/galaxy/tools/parameters/__init__.py       | 24 ++++++------
 lib/galaxy/tools/parameters/meta.py           |  4 +-
 lib/galaxy/tools/toolbox/base.py              |  7 +---
 lib/galaxy/tools/wrappers.py                  |  3 +-
 lib/galaxy/util/permutations.py               |  7 ++--
 lib/galaxy/util/simplegraph.py                |  4 +-
 lib/galaxy/util/tool_shed/common_util.py      |  7 ++--
 lib/galaxy/util/topsort.py                    |  9 ++---
 lib/galaxy/util/yaml_util.py                  |  4 +-
 lib/galaxy/visualization/plugins/registry.py  |  5 +--
 lib/galaxy/web/framework/helpers/grids.py     |  3 +-
 lib/galaxy/web/legacy_framework/grids.py      |  3 +-
 lib/galaxy/webapps/galaxy/api/users.py        |  7 ++--
 .../webapps/galaxy/controllers/admin.py       |  3 +-
 .../webapps/galaxy/controllers/history.py     |  5 +--
 .../webapps/reports/controllers/history.py    |  6 +--
 .../webapps/reports/controllers/tools.py      | 11 +++---
 lib/galaxy/workflow/extract.py                |  3 +-
 lib/galaxy/workflow/modules.py                | 26 ++++++-------
 lib/galaxy/workflow/run.py                    |  3 +-
 .../dependencies/attribute_handlers.py        |  7 ++--
 lib/tool_shed/repository_types/registry.py    |  3 +-
 lib/tool_shed/util/review_util.py             |  3 +-
 .../webapp/controllers/repository_review.py   |  5 +--
 58 files changed, 163 insertions(+), 235 deletions(-)

diff --git a/lib/galaxy/config/config_manage.py b/lib/galaxy/config/config_manage.py
index 238c56cf9d8..b25cafacd46 100644
--- a/lib/galaxy/config/config_manage.py
+++ b/lib/galaxy/config/config_manage.py
@@ -4,9 +4,6 @@ import shutil
 import string
 import sys
 import tempfile
-from collections import (
-    OrderedDict
-)
 from io import StringIO
 from textwrap import TextWrapper
 from typing import Any, List, NamedTuple
@@ -52,7 +49,7 @@ YAML_COMMENT_WRAPPER = TextWrapper(initial_indent="# ", subsequent_indent="# ",
 RST_DESCRIPTION_WRAPPER = TextWrapper(initial_indent="    ", subsequent_indent="    ", break_long_words=False, break_on_hyphens=False)
 UWSGI_SCHEMA_PATH = "lib/galaxy/webapps/uwsgi_schema.yml"
 
-UWSGI_OPTIONS = OrderedDict([
+UWSGI_OPTIONS = dict([
     ('http', {
         'desc': """The address and port on which to listen.  By default, only listen to localhost ($app_name will not be accessible over the network).  Use ':$default_port' to listen on all available network interfaces.""",
         'default': '127.0.0.1:$default_port',
@@ -440,7 +437,7 @@ def _build_uwsgi_schema(args, app_desc):
     last_line = None
     current_opt = None
 
-    options = OrderedDict({})
+    options = {}
     option = None
     for line in rst_options.splitlines():
         line = line.strip()
@@ -505,9 +502,9 @@ def _find_app_options(app_desc, path):
 def _find_app_options_from_config_parser(p):
     if not p.has_section("app:main"):
         _warn(NO_APP_MAIN_MESSAGE)
-        app_items = OrderedDict()
+        app_items = {}
     else:
-        app_items = OrderedDict(p.items("app:main"))
+        app_items = dict(p.items("app:main"))
 
     return app_items
 
@@ -603,9 +600,9 @@ def _run_conversion(args, app_desc):
 
     if not server_section:
         _warn("No server section found, using default uwsgi server definition.")
-        server_config = OrderedDict()
+        server_config = {}
     else:
-        server_config = OrderedDict(p.items(server_section))
+        server_config = dict(p.items(server_section))
 
     app_items = _find_app_options_from_config_parser(p)
     applied_filters = []
@@ -620,7 +617,7 @@ def _run_conversion(args, app_desc):
 
     uwsgi_dict = _server_paste_to_uwsgi(app_desc, server_config, applied_filters)
 
-    app_dict = OrderedDict({})
+    app_dict = {}
     schema = app_desc.schema
     for key, value in app_items.items():
         if key in ["__file__", "here"]:
@@ -777,7 +774,7 @@ def _parse_option_value(option_value):
 
 
 def _server_paste_to_uwsgi(app_desc, server_config, applied_filters):
-    uwsgi_dict = OrderedDict()
+    uwsgi_dict = {}
     port = server_config.get("port", app_desc.default_port)
     host = server_config.get("host", "127.0.0.1")
 
diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py
index b4b76aa62d9..f8dfa4b58bc 100644
--- a/lib/galaxy/datatypes/binary.py
+++ b/lib/galaxy/datatypes/binary.py
@@ -12,7 +12,6 @@ import sys
 import tarfile
 import tempfile
 import zipfile
-from collections import OrderedDict
 from json import dumps
 from typing import Optional
 
@@ -320,7 +319,7 @@ class BamNative(CompressedArchive):
             # TODO: Reference names, lengths, read_groups and headers can become very large, truncate when necessary
             dataset.metadata.reference_names = list(bam_file.references)
             dataset.metadata.reference_lengths = list(bam_file.lengths)
-            dataset.metadata.bam_header = OrderedDict((k, v) for k, v in bam_file.header.items())
+            dataset.metadata.bam_header = dict(bam_file.header.items())
             dataset.metadata.read_groups = [read_group['ID'] for read_group in dataset.metadata.bam_header.get('RG', []) if 'ID' in read_group]
             dataset.metadata.sort_order = dataset.metadata.bam_header.get('HD', {}).get('SO', None)
             dataset.metadata.bam_version = dataset.metadata.bam_header.get('HD', {}).get('VN', None)
diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py
index 175d0f70022..8bf8060535c 100644
--- a/lib/galaxy/datatypes/data.py
+++ b/lib/galaxy/datatypes/data.py
@@ -5,7 +5,6 @@ import os
 import shutil
 import string
 import tempfile
-from collections import OrderedDict
 from inspect import isclass
 from typing import Any, Dict, Optional
 
@@ -125,7 +124,7 @@ class Data(metaclass=DataMeta):
     is_binary = True
     # Composite datatypes
     composite_type: Optional[str] = None
-    composite_files: Dict[str, Any] = OrderedDict()
+    composite_files: Dict[str, Any] = {}
     primary_file_name = 'index'
     # Allow user to change between this datatype and others. If left to None,
     # datatype change is allowed if the datatype is not composite.
@@ -144,7 +143,7 @@ class Data(metaclass=DataMeta):
         object.__init__(self, **kwd)
         self.supported_display_apps = self.supported_display_apps.copy()
         self.composite_files = self.composite_files.copy()
-        self.display_applications = OrderedDict()
+        self.display_applications = {}
 
     @classmethod
     def is_datatype_change_allowed(cls):
@@ -565,7 +564,7 @@ class Data(metaclass=DataMeta):
         return self.display_applications.get(key, default)
 
     def get_display_applications_by_dataset(self, dataset, trans):
-        rval = OrderedDict()
+        rval = {}
         for key, value in self.display_applications.items():
             value = value.filter_by_dataset(dataset, trans)
             if value.links:
@@ -685,7 +684,7 @@ class Data(metaclass=DataMeta):
 
     @property
     def writable_files(self):
-        files = OrderedDict()
+        files = {}
         if self.composite_type != 'auto_primary_file':
             files[self.primary_file_name] = self.__new_composite_file(self.primary_file_name)
         for key, value in self.get_composite_files().items():
@@ -701,7 +700,7 @@ class Data(metaclass=DataMeta):
                     meta_value = self.metadata_spec[composite_file.substitute_name_with_metadata].default
                 return key % meta_value
             return key
-        files = OrderedDict()
+        files = {}
         for key, value in self.composite_files.items():
             files[substitute_composite_key(key, value)] = value
         return files
diff --git a/lib/galaxy/datatypes/display_applications/application.py b/lib/galaxy/datatypes/display_applications/application.py
index f0ee85e1362..f455da7e8ae 100644
--- a/lib/galaxy/datatypes/display_applications/application.py
+++ b/lib/galaxy/datatypes/display_applications/application.py
@@ -1,6 +1,5 @@
 # Contains objects for using external display applications
 import logging
-from collections import OrderedDict
 from copy import deepcopy
 from urllib.parse import quote_plus
 
@@ -46,7 +45,7 @@ class DisplayApplicationLink:
 
     def __init__(self, display_application):
         self.display_application = display_application
-        self.parameters = OrderedDict()  # parameters are populated in order, allowing lower listed ones to have values of higher listed ones
+        self.parameters = {}
         self.url_param_name_map = {}
         self.url = None
         self.id = None
@@ -64,9 +63,9 @@ class DisplayApplicationLink:
 
     def get_inital_values(self, data, trans):
         if self.other_values:
-            rval = OrderedDict(self.other_values)
+            rval = dict(self.other_values)
         else:
-            rval = OrderedDict()
+            rval = {}
         rval.update({'BASE_URL': trans.request.base, 'APP': trans.app})  # trans automatically appears as a response, need to add properties of trans that we want here
         BASE_PARAMS = {'qp': quote_plus_string, 'url_for': trans.app.url_for}
         for key, value in BASE_PARAMS.items():  # add helper functions/variables
@@ -289,7 +288,7 @@ class DisplayApplication:
         if version is None:
             version = "1.0.0"
         self.version = version
-        self.links = OrderedDict()
+        self.links = {}
         self._filename = filename
         self._elem = elem
         self._data_table_versions = {}
diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py
index 379991778c5..99587fe77b5 100644
--- a/lib/galaxy/datatypes/registry.py
+++ b/lib/galaxy/datatypes/registry.py
@@ -5,7 +5,6 @@ Provides mapping between extensions and datatypes, mime-types, etc.
 import imp
 import logging
 import os
-from collections import OrderedDict
 from string import Template
 
 import yaml
@@ -41,7 +40,7 @@ class Registry:
         self.config = config
         self.datatypes_by_extension = {}
         self.mimetypes_by_extension = {}
-        self.datatype_converters = OrderedDict()
+        self.datatype_converters = {}
         # Converters defined in local datatypes_conf.xml
         self.converters = []
         self.converter_tools = set()
@@ -58,7 +57,7 @@ class Registry:
         # tool shed repositories that contain display applications.
         self.proprietary_display_app_containers = []
         # Map a display application id to a display application
-        self.display_applications = OrderedDict()
+        self.display_applications = {}
         # The following 2 attributes are used in the to_xml_file()
         # method to persist the current state into an xml file.
         self.display_path_attr = None
@@ -638,7 +637,7 @@ class Registry:
                 else:
                     toolbox.register_tool(converter)
                     if source_datatype not in self.datatype_converters:
-                        self.datatype_converters[source_datatype] = OrderedDict()
+                        self.datatype_converters[source_datatype] = {}
                     self.datatype_converters[source_datatype][target_datatype] = converter
                     if not hasattr(toolbox.app, 'tool_cache') or converter.id in toolbox.app.tool_cache._new_tool_ids:
                         self.log.debug("Loaded converter: %s", converter.id)
@@ -863,7 +862,7 @@ class Registry:
     def get_converters_by_datatype(self, ext):
         """Returns available converters by source type"""
         if ext not in self._converters_by_datatype:
-            converters = OrderedDict()
+            converters = {}
             source_datatype = type(self.get_datatype_by_extension(ext))
             for ext2, converters_dict in self.datatype_converters.items():
                 converter_datatype = type(self.get_datatype_by_extension(ext2))
diff --git a/lib/galaxy/datatypes/util/gff_util.py b/lib/galaxy/datatypes/util/gff_util.py
index bad68980a01..7656870f02c 100644
--- a/lib/galaxy/datatypes/util/gff_util.py
+++ b/lib/galaxy/datatypes/util/gff_util.py
@@ -2,7 +2,6 @@
 Provides utilities for working with GFF files.
 """
 import copy
-from collections import OrderedDict
 
 from bx.intervals.io import GenomicInterval, GenomicIntervalReader, MissingFieldError, NiceReaderWrapper, ParseError
 from bx.tabular.io import Comment, Header
@@ -428,7 +427,7 @@ def read_unordered_gtf(iterator, strict=False):
             return fields[0] + '_' + get_transcript_id(fields)
 
     # Aggregate intervals by transcript_id and collect comments.
-    feature_intervals = OrderedDict()
+    feature_intervals = {}
     comments = []
     for line in iterator:
         if line.startswith('#'):
diff --git a/lib/galaxy/job_execution/output_collect.py b/lib/galaxy/job_execution/output_collect.py
index 6357310faf1..c4386128d64 100644
--- a/lib/galaxy/job_execution/output_collect.py
+++ b/lib/galaxy/job_execution/output_collect.py
@@ -4,7 +4,6 @@ import logging
 import operator
 import os
 import re
-from collections import OrderedDict
 from tempfile import NamedTemporaryFile
 
 import galaxy.model
@@ -173,7 +172,7 @@ class BaseJobContext:
         pass
 
     def find_files(self, output_name, collection, dataset_collectors):
-        filenames = OrderedDict()
+        filenames = {}
         for discovered_file in discover_files(output_name, self.tool_provided_metadata, dataset_collectors, self.job_working_directory, collection):
             filenames[discovered_file.path] = discovered_file
         return filenames
@@ -373,7 +372,7 @@ def collect_primary_datasets(job_context, output, input_ext):
         output_def = job_context.output_def(name)
         if output_def is not None:
             dataset_collectors = [dataset_collector(description) for description in output_def.dataset_collector_descriptions]
-        filenames = OrderedDict()
+        filenames = {}
         for discovered_file in discover_files(name, job_context.tool_provided_metadata, dataset_collectors, job_working_directory, outdata):
             filenames[discovered_file.path] = discovered_file
         for filename_index, (filename, discovered_file) in enumerate(filenames.items()):
@@ -400,7 +399,7 @@ def collect_primary_datasets(job_context, output, input_ext):
                 primary_output_assigned = True
                 continue
             if name not in primary_datasets:
-                primary_datasets[name] = OrderedDict()
+                primary_datasets[name] = {}
             visible = fields_match.visible
             # Create new primary dataset
             new_primary_name = fields_match.name or f"{outdata.name} ({designation})"
diff --git a/lib/galaxy/managers/collections.py b/lib/galaxy/managers/collections.py
index e1c983aa000..017da542906 100644
--- a/lib/galaxy/managers/collections.py
+++ b/lib/galaxy/managers/collections.py
@@ -1,5 +1,4 @@
 import logging
-from collections import OrderedDict
 
 from sqlalchemy.orm import joinedload, Query
 
@@ -378,7 +377,7 @@ class DatasetCollectionManager:
         if elements is self.ELEMENTS_UNINITIALIZED:
             return
 
-        new_elements = OrderedDict()
+        new_elements = {}
         for key, element in elements.items():
             if isinstance(element, model.DatasetCollection):
                 continue
@@ -387,7 +386,7 @@ class DatasetCollectionManager:
                 continue
 
             # element is a dict with src new_collection and
-            # and OrderedDict of named elements
+            # and dict of named elements
             collection_type = element.get("collection_type")
             sub_elements = element["elements"]
             collection = self.create_dataset_collection(
@@ -402,7 +401,7 @@ class DatasetCollectionManager:
         elements.update(new_elements)
 
     def __load_elements(self, trans, element_identifiers, hide_source_items=False, copy_elements=False, history=None):
-        elements = OrderedDict()
+        elements = {}
         for element_identifier in element_identifiers:
             elements[element_identifier["name"]] = self.__load_element(trans,
                                                                        element_identifier=element_identifier,
@@ -500,7 +499,7 @@ class DatasetCollectionManager:
     def _build_elements_from_rule_data(self, collection_type_description, rule_set, data, sources, handle_dataset):
         identifier_columns = rule_set.identifier_columns
         mapping_as_dict = rule_set.mapping_as_dict
-        elements = OrderedDict()
+        elements = {}
         for data_index, row_data in enumerate(data):
             # For each row, find place in depth for this element.
             collection_type_at_depth = collection_type_description
@@ -546,7 +545,7 @@ class DatasetCollectionManager:
                         sub_collection = {}
                         sub_collection["src"] = "new_collection"
                         sub_collection["collection_type"] = collection_type_at_depth.collection_type
-                        sub_collection["elements"] = OrderedDict()
+                        sub_collection["elements"] = {}
                         elements_at_depth[identifier] = sub_collection
                         elements_at_depth = sub_collection["elements"]
 
diff --git a/lib/galaxy/managers/executables.py b/lib/galaxy/managers/executables.py
index 7d391a0179b..d4250691c21 100644
--- a/lib/galaxy/managers/executables.py
+++ b/lib/galaxy/managers/executables.py
@@ -1,6 +1,6 @@
 """Utilities for loading tools and workflows from paths for admin user requests."""
 
-from gxformat2.converter import ordered_load
+import yaml
 
 from galaxy import exceptions
 
@@ -13,7 +13,7 @@ def artifact_class(trans, as_dict):
 
         workflow_path = as_dict.get("path")
         with open(workflow_path) as f:
-            as_dict = ordered_load(f)
+            as_dict = yaml.safe_load(f)
 
     artifact_class = as_dict.get("class", None)
     if artifact_class is None and "$graph" in as_dict:
diff --git a/lib/galaxy/managers/markdown_util.py b/lib/galaxy/managers/markdown_util.py
index 43e0a48f8f9..847a8cb4e5f 100644
--- a/lib/galaxy/managers/markdown_util.py
+++ b/lib/galaxy/managers/markdown_util.py
@@ -18,7 +18,6 @@ import os
 import re
 import shutil
 import tempfile
-from collections import OrderedDict
 
 import markdown
 import pkg_resources
@@ -444,11 +443,11 @@ class ToBasicMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHandler):
 
     def handle_job_metrics(self, line, job):
         job_metrics = summarize_job_metrics(self.trans, job)
-        metrics_by_plugin = OrderedDict()
+        metrics_by_plugin = {}
         for job_metric in job_metrics:
             plugin = job_metric["plugin"]
             if plugin not in metrics_by_plugin:
-                metrics_by_plugin[plugin] = OrderedDict()
+                metrics_by_plugin[plugin] = {}
             metrics_by_plugin[plugin][job_metric["title"]] = job_metric["value"]
         markdown = ""
         for metric_plugin, metrics_for_plugin in metrics_by_plugin.items():
diff --git a/lib/galaxy/model/dataset_collections/builder.py b/lib/galaxy/model/dataset_collections/builder.py
index 0071cf30127..05694c34289 100644
--- a/lib/galaxy/model/dataset_collections/builder.py
+++ b/lib/galaxy/model/dataset_collections/builder.py
@@ -1,5 +1,3 @@
-from collections import OrderedDict
-
 from galaxy import model
 from .type_description import COLLECTION_TYPE_DESCRIPTION_FACTORY
 
@@ -34,7 +32,7 @@ class CollectionBuilder:
 
     def __init__(self, collection_type_description):
         self._collection_type_description = collection_type_description
-        self._current_elements = OrderedDict()
+        self._current_elements = {}
 
     def replace_elements_in_collection(self, template_collection, replacement_dict):
         self._current_elements = self._replace_elements_in_collection(
@@ -43,7 +41,7 @@ class CollectionBuilder:
         )
 
     def _replace_elements_in_collection(self, template_collection, replacement_dict):
-        elements = OrderedDict()
+        elements = {}
         for element in template_collection.elements:
             if element.is_collection:
                 collection_builder = CollectionBuilder(
@@ -77,7 +75,7 @@ class CollectionBuilder:
     def build_elements(self):
         elements = self._current_elements
         if self._nested_collection:
-            new_elements = OrderedDict()
+            new_elements = {}
             for identifier, element in elements.items():
                 new_elements[identifier] = element.build()
             elements = new_elements
diff --git a/lib/galaxy/model/store/discover.py b/lib/galaxy/model/store/discover.py
index 6d61479b47e..07ea899ce20 100644
--- a/lib/galaxy/model/store/discover.py
+++ b/lib/galaxy/model/store/discover.py
@@ -10,7 +10,6 @@ import logging
 import os
 from collections import (
     namedtuple,
-    OrderedDict
 )
 from typing import Any, NamedTuple, Optional
 
@@ -517,7 +516,7 @@ def persist_target_to_export_store(target_dict, export_store, object_store, work
 
 
 def persist_elements_to_hdca(model_persistence_context, elements, hdca, collector=None):
-    filenames = OrderedDict()
+    filenames = {}
 
     def add_to_discovered_files(elements, parent_identifiers=None):
         parent_identifiers = parent_identifiers or []
diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py
index d967c0df1a8..e95383d8c72 100644
--- a/lib/galaxy/objectstore/__init__.py
+++ b/lib/galaxy/objectstore/__init__.py
@@ -12,7 +12,6 @@ import random
 import shutil
 import threading
 import time
-from collections import OrderedDict
 
 import yaml
 
@@ -932,7 +931,7 @@ class HierarchicalObjectStore(NestedObjectStore):
         """The default constructor. Extends `NestedObjectStore`."""
         super().__init__(config, config_dict)
 
-        backends = OrderedDict()
+        backends = {}
         for order, backend_def in enumerate(config_dict["backends"]):
             backends[order] = build_object_store_from_config(config, config_dict=backend_def, fsmon=fsmon)
 
diff --git a/lib/galaxy/openid/providers.py b/lib/galaxy/openid/providers.py
index 5f00205518f..b3f695ba6b6 100644
--- a/lib/galaxy/openid/providers.py
+++ b/lib/galaxy/openid/providers.py
@@ -3,7 +3,6 @@ Contains OpenID provider functionality
 """
 import logging
 import os
-from collections import OrderedDict
 
 
 from galaxy.util import parse_xml, string_as_bool
@@ -115,7 +114,7 @@ class OpenIDProviders:
     @classmethod
     def from_elem(cls, xml_root):
         oid_elem = xml_root
-        providers = OrderedDict()
+        providers = {}
         for elem in oid_elem.findall('provider'):
             try:
                 provider = OpenIDProvider.from_file(os.path.join('lib/galaxy/openid', elem.get('file')))
@@ -129,7 +128,7 @@ class OpenIDProviders:
         if providers:
             self.providers = providers
         else:
-            self.providers = OrderedDict()
+            self.providers = {}
         self._banned_identifiers = [provider.op_endpoint_url for provider in self.providers.values() if provider.never_associate_with_user]
 
     def __iter__(self):
diff --git a/lib/galaxy/tool_shed/galaxy_install/migrate/check.py b/lib/galaxy/tool_shed/galaxy_install/migrate/check.py
index 577d7b47eba..f07ab310b45 100644
--- a/lib/galaxy/tool_shed/galaxy_install/migrate/check.py
+++ b/lib/galaxy/tool_shed/galaxy_install/migrate/check.py
@@ -2,7 +2,6 @@ import logging
 import os
 import subprocess
 import sys
-from collections import OrderedDict
 
 from migrate.versioning import repository, schema
 from sqlalchemy import create_engine, MetaData, Table
@@ -34,7 +33,7 @@ def verify_tools(app, url, galaxy_config_file=None, engine_options=None):
         tool_shed_accessible = False
         if app.new_installation:
             # New installations will not be missing tools, so we don't need to worry about them.
-            missing_tool_configs_dict = OrderedDict()
+            missing_tool_configs_dict = {}
         else:
             tool_panel_configs = common_util.get_non_shed_tool_panel_configs(app)
             if tool_panel_configs:
@@ -48,7 +47,7 @@ def verify_tools(app, url, galaxy_config_file=None, engine_options=None):
                 # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in
                 # the database.
                 tool_shed_accessible = True
-                missing_tool_configs_dict = OrderedDict()
+                missing_tool_configs_dict = {}
         have_tool_dependencies = False
         for v in missing_tool_configs_dict.values():
             if v:
diff --git a/lib/galaxy/tool_shed/galaxy_install/tool_migration_manager.py b/lib/galaxy/tool_shed/galaxy_install/tool_migration_manager.py
index 5ed88b9996b..8e39c3fe002 100644
--- a/lib/galaxy/tool_shed/galaxy_install/tool_migration_manager.py
+++ b/lib/galaxy/tool_shed/galaxy_install/tool_migration_manager.py
@@ -8,7 +8,6 @@ import os
 import shutil
 import tempfile
 import threading
-from collections import OrderedDict
 
 from galaxy import util
 from galaxy.tool_shed.galaxy_install import install_manager
@@ -113,7 +112,7 @@ class ToolMigrationManager:
                     # tool_shed_accessible to True so that the value of migrate_tools.version can
                     # be correctly set in the database.
                     tool_shed_accessible = True
-                    missing_tool_configs_dict = OrderedDict()
+                    missing_tool_configs_dict = {}
                 if tool_shed_accessible:
                     if len(self.proprietary_tool_confs) == 1:
                         plural = ''
@@ -387,7 +386,7 @@ class ToolMigrationManager:
         entries are automatically added to the reserved migrated_tools_conf.xml file as part of the migration process.
         """
         tool_configs_to_filter = []
-        tool_panel_dict_for_display = OrderedDict()
+        tool_panel_dict_for_display = {}
         if self.tool_path:
             repo_install_dir = os.path.join(self.tool_path, relative_install_dir)
         else:
diff --git a/lib/galaxy/tool_shed/tool_shed_registry.py b/lib/galaxy/tool_shed/tool_shed_registry.py
index 3022b435714..f20c58eb3eb 100644
--- a/lib/galaxy/tool_shed/tool_shed_registry.py
+++ b/lib/galaxy/tool_shed/tool_shed_registry.py
@@ -1,8 +1,5 @@
 import logging
-from collections import (
-    namedtuple,
-    OrderedDict,
-)
+from collections import namedtuple
 
 from galaxy.util import parse_xml_string
 from galaxy.util.tool_shed.common_util import remove_protocol_from_tool_shed_url
@@ -22,8 +19,8 @@ AUTH_TUPLE = namedtuple('AuthSetting', 'username password')
 class Registry:
 
     def __init__(self, config=None):
-        self.tool_sheds = OrderedDict()
-        self.tool_sheds_auth = OrderedDict()
+        self.tool_sheds = {}
+        self.tool_sheds_auth = {}
         if config:
             # Parse tool_sheds_conf.xml
             tree, error_message = parse_xml(config)
diff --git a/lib/galaxy/tool_util/cwl/parser.py b/lib/galaxy/tool_util/cwl/parser.py
index df518dae663..70661aa3bf0 100644
--- a/lib/galaxy/tool_util/cwl/parser.py
+++ b/lib/galaxy/tool_util/cwl/parser.py
@@ -11,7 +11,6 @@ import logging
 import os
 import pickle
 from abc import ABCMeta, abstractmethod
-from collections import OrderedDict
 from uuid import uuid4
 
 
@@ -1214,7 +1213,7 @@ class ConditionalInstance:
             name=self.name,
             type=INPUT_TYPE.CONDITIONAL,
             test=self.case.to_dict(),
-            when=OrderedDict(),
+            when={},
         )
         for value, block in self.whens:
             as_dict["when"][value] = [i.to_dict() for i in block]
diff --git a/lib/galaxy/tool_util/cwl/representation.py b/lib/galaxy/tool_util/cwl/representation.py
index d428db4fa39..9dc01f1ef68 100644
--- a/lib/galaxy/tool_util/cwl/representation.py
+++ b/lib/galaxy/tool_util/cwl/representation.py
@@ -1,7 +1,6 @@
 """ This module is responsible for converting between Galaxy's tool
 input description and the CWL description for a job json. """
 
-import collections
 import json
 import logging
 import os
@@ -210,7 +209,7 @@ def collection_wrapper_to_array(inputs_dir, wrapped_value):
 
 
 def collection_wrapper_to_record(inputs_dir, wrapped_value):
-    rval = collections.OrderedDict()
+    rval = {}
     for key, value in wrapped_value.items():
         rval[key] = dataset_wrapper_to_file_json(inputs_dir, value)
     return rval
diff --git a/lib/galaxy/tool_util/deps/__init__.py b/lib/galaxy/tool_util/deps/__init__.py
index 1a03f5a6bda..753ea0cf2a5 100644
--- a/lib/galaxy/tool_util/deps/__init__.py
+++ b/lib/galaxy/tool_util/deps/__init__.py
@@ -6,7 +6,6 @@ import json
 import logging
 import os.path
 import shutil
-from collections import OrderedDict
 
 from galaxy.util import (
     hash_util,
@@ -201,7 +200,7 @@ class DependencyManager:
 
     def _requirements_to_dependencies_dict(self, requirements, search=False, **kwds):
         """Build simple requirements to dependencies dict for resolution."""
-        requirement_to_dependency = OrderedDict()
+        requirement_to_dependency = {}
         index = kwds.get('index')
         install = kwds.get('install', False)
         resolver_type = kwds.get('resolver_type')
@@ -233,7 +232,7 @@ class DependencyManager:
             if container_type is not None and getattr(resolver, "container_type", None) != container_type:
                 continue
 
-            _requirement_to_dependency = OrderedDict([(k, v) for k, v in requirement_to_dependency.items() if not isinstance(v, NullDependency)])
+            _requirement_to_dependency = {k: v for k, v in requirement_to_dependency.items() if not isinstance(v, NullDependency)}
 
             if len(_requirement_to_dependency) == len(resolvable_requirements):
                 # Shortcut - resolution complete.
diff --git a/lib/galaxy/tool_util/parser/cwl.py b/lib/galaxy/tool_util/parser/cwl.py
index cfdc62dd304..0ebb4a48261 100644
--- a/lib/galaxy/tool_util/parser/cwl.py
+++ b/lib/galaxy/tool_util/parser/cwl.py
@@ -1,6 +1,5 @@
 import logging
 import os
-from collections import OrderedDict
 
 from galaxy.tool_util.cwl.parser import tool_proxy
 from galaxy.tool_util.deps import requirements
@@ -113,14 +112,14 @@ class CwlToolSource(ToolSource):
 
     def parse_outputs(self, tool):
         output_instances = self.tool_proxy.output_instances()
-        outputs = OrderedDict()
+        outputs = {}
         output_defs = []
         for output_instance in output_instances:
             output_defs.append(self._parse_output(tool, output_instance))
         # TODO: parse outputs collections
         for output_def in output_defs:
             outputs[output_def.name] = output_def
-        return outputs, OrderedDict()
+        return outputs, {}
 
     def _parse_output(self, tool, output_instance):
         name = output_instance.name
diff --git a/lib/galaxy/tool_util/parser/output_objects.py b/lib/galaxy/tool_util/parser/output_objects.py
index 1827f2cc710..2750c081714 100644
--- a/lib/galaxy/tool_util/parser/output_objects.py
+++ b/lib/galaxy/tool_util/parser/output_objects.py
@@ -1,5 +1,3 @@
-from collections import OrderedDict
-
 from galaxy.util.dictifiable import Dictifiable
 from .output_actions import ToolOutputActionGroup
 from .output_collection_def import dataset_collector_descriptions_from_output_dict
@@ -156,7 +154,7 @@ class ToolOutputCollection(ToolOutputBase):
         self.collection = True
         self.default_format = default_format
         self.structure = structure
-        self.outputs = OrderedDict()
+        self.outputs = {}
 
         self.inherit_format = inherit_format
         self.inherit_metadata = inherit_metadata
diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py
index e5573820c45..0355946ab27 100644
--- a/lib/galaxy/tool_util/parser/xml.py
+++ b/lib/galaxy/tool_util/parser/xml.py
@@ -2,7 +2,6 @@ import json
 import logging
 import re
 import uuid
-from collections import OrderedDict
 from math import isinf
 
 import packaging.version
@@ -290,12 +289,12 @@ class XmlToolSource(ToolSource):
 
     def parse_outputs(self, tool):
         out_elem = self.root.find("outputs")
-        outputs = OrderedDict()
-        output_collections = OrderedDict()
+        outputs = {}
+        output_collections = {}
         if out_elem is None:
             return outputs, output_collections
 
-        data_dict = OrderedDict()
+        data_dict = {}
 
         def _parse(data_elem, **kwds):
             output_def = self._parse_output(data_elem, tool, **kwds)
diff --git a/lib/galaxy/tool_util/parser/yaml.py b/lib/galaxy/tool_util/parser/yaml.py
index a8c5485dde0..5baec304acd 100644
--- a/lib/galaxy/tool_util/parser/yaml.py
+++ b/lib/galaxy/tool_util/parser/yaml.py
@@ -1,5 +1,3 @@
-from collections import OrderedDict
-
 import packaging.version
 
 from galaxy.tool_util.deps import requirements
@@ -117,10 +115,10 @@ class YamlToolSource(ToolSource):
             else:
                 message = "Unknown output_type [%s] encountered." % output_type
                 raise Exception(message)
-        outputs = OrderedDict()
+        outputs = {}
         for output in output_defs:
             outputs[output.name] = output
-        output_collections = OrderedDict()
+        output_collections = {}
         for output in output_collection_defs:
             output_collections[output.name] = output
 
diff --git a/lib/galaxy/tool_util/verify/interactor.py b/lib/galaxy/tool_util/verify/interactor.py
index 0c110414a78..88f0d375bb8 100644
--- a/lib/galaxy/tool_util/verify/interactor.py
+++ b/lib/galaxy/tool_util/verify/interactor.py
@@ -8,7 +8,6 @@ import tarfile
 import tempfile
 import time
 import zipfile
-from collections import OrderedDict
 from json import dumps
 from logging import getLogger
 
@@ -43,7 +42,7 @@ DEFAULT_FTYPE = 'auto'
 DEFAULT_DBKEY = os.environ.get("GALAXY_TEST_DEFAULT_DBKEY", "?")
 
 
-class OutputsDict(OrderedDict):
+class OutputsDict(dict):
     """Ordered dict that can also be accessed by index.
 
     >>> out = OutputsDict()
@@ -57,12 +56,7 @@ class OutputsDict(OrderedDict):
         if isinstance(item, int):
             return self[list(self.keys())[item]]
         else:
-            # ideally we'd do `return super(OutputsDict, self)[item]`,
-            # but this fails because OrderedDict has no `__getitem__`. (!?)
-            item = self.get(item)
-            if item is None:
-                raise KeyError(item)
-            return item
+            return super().__getitem__(item)
 
 
 def stage_data_in_history(galaxy_interactor, tool_id, all_test_data, history=None, force_path_paste=False, maxseconds=DEFAULT_TOOL_TEST_WAIT):
@@ -497,7 +491,7 @@ class GalaxyInteractorApi:
         return element_identifiers
 
     def __dictify_output_collections(self, submit_response):
-        output_collections_dict = OrderedDict()
+        output_collections_dict = {}
         for output_collection in submit_response['output_collections']:
             output_collections_dict[output_collection.get("output_name")] = output_collection
         return output_collections_dict
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 13c89d9b7ff..98f2cb19045 100644
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -10,7 +10,6 @@ import re
 import tarfile
 import tempfile
 import threading
-from collections import OrderedDict
 from datetime import datetime
 from pathlib import Path
 from typing import List, Type
@@ -1123,7 +1122,7 @@ class Tool(Dictifiable):
         This implementation supports multiple pages and grouping constructs.
         """
         # Load parameters (optional)
-        self.inputs = OrderedDict()
+        self.inputs = {}
         pages = tool_source.parse_input_pages()
         enctypes = set()
         if pages.inputs_defined:
@@ -1236,7 +1235,7 @@ class Tool(Dictifiable):
         groups (repeat, conditional) or param elements. Groups will be parsed
         recursively.
         """
-        rval = OrderedDict()
+        rval = {}
         context = ExpressionContext(rval, context)
         for input_source in page_source.parse_input_sources():
             # Repeat group
@@ -1276,7 +1275,7 @@ class Tool(Dictifiable):
                             page_source = XmlPageSource(XML("<when>%s</when>" % case_inputs))
                             case.inputs = self.parse_input_elem(page_source, enctypes, context)
                         else:
-                            case.inputs = OrderedDict()
+                            case.inputs = {}
                         group.cases.append(case)
                 else:
                     # Should have one child "input" which determines the case
@@ -1304,7 +1303,7 @@ class Tool(Dictifiable):
                                     (self.id, group.name, group.test_param.name, unspecified_case))
                         case = ConditionalWhen()
                         case.value = unspecified_case
-                        case.inputs = OrderedDict()
+                        case.inputs = {}
                         group.cases.append(case)
                 rval[group.name] = group
             elif input_type == "section":
@@ -1672,7 +1671,7 @@ class Tool(Dictifiable):
             log.exception("Exception caught while attempting to execute tool with id '%s':", self.id)
             message = "Error executing tool with id '{}': {}".format(self.id, unicodify(e))
             return False, message
-        if isinstance(out_data, OrderedDict):
+        if isinstance(out_data, dict):
             return job, list(out_data.items())
         else:
             if isinstance(out_data, str):
@@ -2806,7 +2805,7 @@ class DatabaseOperationTool(Tool):
         return self._outputs_dict()
 
     def _outputs_dict(self):
-        return OrderedDict()
+        return {}
 
 
 class UnzipCollectionTool(DatabaseOperationTool):
@@ -2838,7 +2837,7 @@ class ZipCollectionTool(DatabaseOperationTool):
         reverse_o = incoming["input_reverse"]
 
         forward, reverse = forward_o.copy(), reverse_o.copy()
-        new_elements = OrderedDict()
+        new_elements = {}
         new_elements["forward"] = forward
         new_elements["reverse"] = reverse
         self._add_datasets_to_history(history, [forward, reverse])
@@ -2851,7 +2850,7 @@ class BuildListCollectionTool(DatabaseOperationTool):
     tool_type = 'build_list'
 
     def produce_outputs(self, trans, out_data, output_collections, incoming, history, tags=None, **kwds):
-        new_elements = OrderedDict()
+        new_elements = {}
 
         for i, incoming_repeat in enumerate(incoming["datasets"]):
             if incoming_repeat["input"]:
@@ -2911,7 +2910,7 @@ class MergeCollectionTool(DatabaseOperationTool):
             if dupl_actions in ['suffix_conflict', 'suffix_every', 'suffix_conflict_rest']:
                 suffix_pattern = advanced['conflict']['suffix_pattern']
 
-        new_element_structure = OrderedDict()
+        new_element_structure = {}
 
         # Which inputs does the identifier appear in.
         identifiers_map = {}
@@ -2963,7 +2962,7 @@ class MergeCollectionTool(DatabaseOperationTool):
                     new_element_structure[effective_identifer] = element
 
         # Don't copy until we know everything is fine and we have the structure of the list ready to go.
-        new_elements = OrderedDict()
+        new_elements = {}
         for key, value in new_element_structure.items():
             if getattr(value, "history_content_type", None) == "dataset":
                 copied_value = value.copy(flush=False)
@@ -2980,7 +2979,7 @@ class MergeCollectionTool(DatabaseOperationTool):
 class FilterDatasetsTool(DatabaseOperationTool):
 
     def _get_new_elements(self, history, elements_to_copy):
-        new_elements = OrderedDict()
+        new_elements = {}
         for dce in elements_to_copy:
             element_identifier = dce.element_identifier
             if getattr(dce.element_object, "history_content_type", None) == "dataset":
@@ -3048,7 +3047,7 @@ class FlattenTool(DatabaseOperationTool):
     def produce_outputs(self, trans, out_data, output_collections, incoming, history, **kwds):
         hdca = incoming["input"]
         join_identifier = incoming["join_identifier"]
-        new_elements = OrderedDict()
+        new_elements = {}
         copied_datasets = []
 
         def add_elements(collection, prefix=""):
@@ -3076,7 +3075,7 @@ class SortTool(DatabaseOperationTool):
     def produce_outputs(self, trans, out_data, output_collections, incoming, history, **kwds):
         hdca = incoming["input"]
         sorttype = incoming["sort_type"]["sort_type"]
-        new_elements = OrderedDict()
+        new_elements = {}
         elements = hdca.collection.elements
         presort_elements = []
         if sorttype == 'alpha':
@@ -3089,7 +3088,7 @@ class SortTool(DatabaseOperationTool):
             hda = incoming["sort_type"]["sort_file"]
             data_lines = hda.metadata.get('data_lines', 0)
             if data_lines == len(elements):
-                old_elements_dict = OrderedDict()
+                old_elements_dict = {}
                 for element in elements:
                     old_elements_dict[element.element_identifier] = element
                 try:
@@ -3122,7 +3121,7 @@ class RelabelFromFileTool(DatabaseOperationTool):
         how_type = incoming["how"]["how_select"]
         new_labels_dataset_assoc = incoming["how"]["labels"]
         strict = string_as_bool(incoming["how"]["strict"])
-        new_elements = OrderedDict()
+        new_elements = {}
 
         def add_copied_value_to_new_elements(new_label, dce_object):
             new_label = new_label.strip()
@@ -3198,7 +3197,7 @@ class TagFromFileTool(DatabaseOperationTool):
         hdca = incoming["input"]
         how = incoming['how']
         new_tags_dataset_assoc = incoming["tags"]
-        new_elements = OrderedDict()
+        new_elements = {}
         new_datasets = []
 
         def add_copied_value_to_new_elements(new_tags_dict, dce):
@@ -3259,8 +3258,8 @@ class FilterFromFileTool(DatabaseOperationTool):
         hdca = incoming["input"]
         how_filter = incoming["how"]["how_filter"]
         filter_dataset_assoc = incoming["how"]["filter_source"]
-        filtered_elements = OrderedDict()
-        discarded_elements = OrderedDict()
+        filtered_elements = {}
+        discarded_elements = {}
 
         filtered_path = filter_dataset_assoc.file_name
         with open(filtered_path) as fh:
diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py
index afb8cf43681..1f999877b4a 100644
--- a/lib/galaxy/tools/actions/__init__.py
+++ b/lib/galaxy/tools/actions/__init__.py
@@ -2,7 +2,6 @@ import json
 import logging
 import os
 import re
-from collections import OrderedDict
 from json import dumps
 
 
@@ -69,7 +68,7 @@ class DefaultToolAction:
         """
         if current_user_roles is None:
             current_user_roles = trans.get_current_user_roles()
-        input_datasets = OrderedDict()
+        input_datasets = {}
         all_permissions = {}
 
         def record_permission(action, role_id):
@@ -357,7 +356,7 @@ class DefaultToolAction:
         # wrapped params are used by change_format action and by output.label; only perform this wrapping once, as needed
         wrapped_params = self._wrapped_params(trans, tool, incoming, inp_data)
 
-        out_data = OrderedDict()
+        out_data = {}
         input_collections = {k: v[0][0] for k, v in inp_dataset_collections.items()}
         output_collections = OutputCollections(
             trans,
@@ -852,7 +851,7 @@ class OutputCollections:
                     # We don't care about the repeat index, we just need to find the correct DataCollectionToolParameter
                 else:
                     key = group
-                if isinstance(data_param, OrderedDict):
+                if isinstance(data_param, dict):
                     data_param = data_param.get(key)
                 else:
                     data_param = data_param.inputs.get(key)
diff --git a/lib/galaxy/tools/actions/history_imp_exp.py b/lib/galaxy/tools/actions/history_imp_exp.py
index c0ba494fc68..5b881b96820 100644
--- a/lib/galaxy/tools/actions/history_imp_exp.py
+++ b/lib/galaxy/tools/actions/history_imp_exp.py
@@ -2,7 +2,6 @@ import datetime
 import logging
 import os
 import tempfile
-from collections import OrderedDict
 
 from galaxy.job_execution.setup import create_working_directory_for_job
 from galaxy.tools.actions import ToolAction
@@ -73,7 +72,7 @@ class ImportHistoryToolAction(ToolAction):
         trans.app.job_manager.enqueue(job, tool=tool)
         trans.log_event("Added import history job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id)
 
-        return job, OrderedDict()
+        return job, {}
 
 
 class ExportHistoryToolAction(ToolAction):
@@ -173,4 +172,4 @@ class ExportHistoryToolAction(ToolAction):
         trans.app.job_manager.enqueue(job, tool=tool)
         trans.log_event("Added export history job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id)
 
-        return job, OrderedDict()
+        return job, {}
diff --git a/lib/galaxy/tools/actions/metadata.py b/lib/galaxy/tools/actions/metadata.py
index 59f3d806c5a..33647c5104c 100644
--- a/lib/galaxy/tools/actions/metadata.py
+++ b/lib/galaxy/tools/actions/metadata.py
@@ -1,6 +1,5 @@
 import logging
 import os
-from collections import OrderedDict
 from json import dumps
 
 from galaxy.job_execution.datasets import DatasetPath
@@ -129,4 +128,4 @@ class SetMetadataToolAction(ToolAction):
         # clear e.g. converted files
         dataset.datatype.before_setting_metadata(dataset)
 
-        return job, OrderedDict()
+        return job, {}
diff --git a/lib/galaxy/tools/actions/model_operations.py b/lib/galaxy/tools/actions/model_operations.py
index 3cae2c69ce2..c152bb81e1c 100644
--- a/lib/galaxy/tools/actions/model_operations.py
+++ b/lib/galaxy/tools/actions/model_operations.py
@@ -1,5 +1,4 @@
 import logging
-from collections import OrderedDict
 
 from galaxy.tools.actions import (
     DefaultToolAction,
@@ -38,7 +37,7 @@ class ModelOperationToolAction(DefaultToolAction):
         # wrapped params are used by change_format action and by output.label; only perform this wrapping once, as needed
         wrapped_params = self._wrapped_params(trans, tool, incoming)
 
-        out_data = OrderedDict()
+        out_data = {}
         input_collections = {k: v[0][0] for k, v in inp_dataset_collections.items()}
         output_collections = OutputCollections(
             trans,
diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py
index ce763c00c6c..0fea6c4ba14 100644
--- a/lib/galaxy/tools/actions/upload_common.py
+++ b/lib/galaxy/tools/actions/upload_common.py
@@ -3,7 +3,6 @@ import logging
 import os
 import socket
 import tempfile
-from collections import OrderedDict
 from io import StringIO
 from json import dump, dumps
 from urllib.parse import urlparse
@@ -442,7 +441,7 @@ def create_job(trans, params, tool, json_file_path, outputs, folder=None, histor
     # Queue the job for execution
     trans.app.job_manager.enqueue(job, tool=tool)
     trans.log_event("Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id)
-    output = OrderedDict()
+    output = {}
     for i, v in enumerate(outputs):
         if not hasattr(output_object, "collection_type"):
             output['output%i' % i] = v
diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py
index 07fe2e82de5..ce18a99177a 100644
--- a/lib/galaxy/tools/data/__init__.py
+++ b/lib/galaxy/tools/data/__init__.py
@@ -14,7 +14,6 @@ import os.path
 import re
 import string
 import time
-from collections import OrderedDict
 from glob import glob
 from tempfile import NamedTemporaryFile
 from typing import List
@@ -256,7 +255,7 @@ class ToolDataTable:
         self.empty_field_values = {}
         self.allow_duplicate_entries = util.asbool(config_element.get('allow_duplicate_entries', True))
         self.here = filename and os.path.dirname(filename)
-        self.filenames = OrderedDict()
+        self.filenames = {}
         self.tool_data_path = tool_data_path
         self.tool_data_path_files = tool_data_path_files
         self.other_config_dict = other_config_dict or {}
diff --git a/lib/galaxy/tools/data_manager/manager.py b/lib/galaxy/tools/data_manager/manager.py
index bfa3e7ba073..a8891249288 100644
--- a/lib/galaxy/tools/data_manager/manager.py
+++ b/lib/galaxy/tools/data_manager/manager.py
@@ -2,7 +2,6 @@ import errno
 import json
 import logging
 import os
-from collections import OrderedDict
 
 
 from galaxy import util
@@ -19,8 +18,8 @@ DEFAULT_VALUE_TRANSLATION_TYPE = 'template'
 class DataManagers:
     def __init__(self, app, xml_filename=None):
         self.app = app
-        self.data_managers = OrderedDict()
-        self.managed_data_tables = OrderedDict()
+        self.data_managers = {}
+        self.managed_data_tables = {}
         self.tool_path = None
         self._reload_count = 0
         self.filename = xml_filename or self.app.config.data_manager_config_file
@@ -123,7 +122,7 @@ class DataManager:
         self.version = self.DEFAULT_VERSION
         self.guid = None
         self.tool = None
-        self.data_tables = OrderedDict()
+        self.data_tables = {}
         self.output_ref_by_data_table = {}
         self.move_by_data_table_column = {}
         self.value_translation_by_data_table_column = {}
@@ -171,7 +170,7 @@ class DataManager:
             data_table_name = data_table_elem.get("name")
             assert data_table_name is not None, "A name is required for a data table entry"
             if data_table_name not in self.data_tables:
-                self.data_tables[data_table_name] = OrderedDict()
+                self.data_tables[data_table_name] = {}
             output_elem = data_table_elem.find('output')
             if output_elem is not None:
                 for column_elem in output_elem.findall('column'):
diff --git a/lib/galaxy/tools/execute.py b/lib/galaxy/tools/execute.py
index a142adaaa80..70f6ec56a5c 100644
--- a/lib/galaxy/tools/execute.py
+++ b/lib/galaxy/tools/execute.py
@@ -152,7 +152,7 @@ class ExecutionTracker:
         self.output_datasets = []
         self.output_collections = []
 
-        self.implicit_collections = collections.OrderedDict()
+        self.implicit_collections = {}
 
     @property
     def param_combinations(self):
diff --git a/lib/galaxy/tools/parameters/__init__.py b/lib/galaxy/tools/parameters/__init__.py
index c9750b83dd1..7fa20bbd13b 100644
--- a/lib/galaxy/tools/parameters/__init__.py
+++ b/lib/galaxy/tools/parameters/__init__.py
@@ -27,7 +27,6 @@ def visit_input_values(inputs, input_values, callback, name_prefix='', label_pre
 
     If the callback returns a value, it will be replace the old value.
 
-    >>> from collections import OrderedDict
     >>> from galaxy.util import XML
     >>> from galaxy.util.bunch import Bunch
     >>> from galaxy.tools.parameters.basic import TextToolParameter, BooleanToolParameter
@@ -43,9 +42,9 @@ def visit_input_values(inputs, input_values, callback, name_prefix='', label_pre
     >>> i = TextToolParameter(None, XML('<param name="i"/>'))
     >>> j = TextToolParameter(None, XML('<param name="j"/>'))
     >>> b.name = b.title = 'b'
-    >>> b.inputs = OrderedDict([ ('c', c), ('d', d) ])
+    >>> b.inputs = dict([ ('c', c), ('d', d) ])
     >>> d.name = d.title = 'd'
-    >>> d.inputs = OrderedDict([ ('e', e), ('f', f) ])
+    >>> d.inputs = dict([ ('e', e), ('f', f) ])
     >>> f.test_param = g
     >>> f.name = 'f'
     >>> f.cases = [Bunch(value='true', inputs= {'h': h}), Bunch(value='false', inputs= { 'i': i })]
@@ -54,8 +53,8 @@ def visit_input_values(inputs, input_values, callback, name_prefix='', label_pre
     ...     print('name=%s, prefix=%s, prefixed_name=%s, prefixed_label=%s, value=%s' % (input.name, prefix, prefixed_name, prefixed_label, value))
     ...     if error:
     ...         print(error)
-    >>> inputs = OrderedDict([('a', a),('b', b)])
-    >>> nested = OrderedDict([('a', 1), ('b', [OrderedDict([('c', 3), ('d', [OrderedDict([ ('e', 5), ('f', OrderedDict([ ('g', True), ('h', 7)]))])])])])])
+    >>> inputs = dict([('a', a),('b', b)])
+    >>> nested = dict([('a', 1), ('b', [dict([('c', 3), ('d', [dict([ ('e', 5), ('f', dict([ ('g', True), ('h', 7)]))])])])])])
     >>> visit_input_values(inputs, nested, visitor)
     name=a, prefix=, prefixed_name=a, prefixed_label=a, value=1
     name=c, prefix=b_0|, prefixed_name=b_0|c, prefixed_label=b 1 > c, value=3
@@ -103,7 +102,7 @@ def visit_input_values(inputs, input_values, callback, name_prefix='', label_pre
     No value found for 'b 1 > d 1 > j'.
 
     >>> # Other parameters are missing in state
-    >>> nested = OrderedDict([('b', [OrderedDict([( 'd', [OrderedDict([('f', OrderedDict([('g', True), ('h', 7)]))])])])])])
+    >>> nested = dict([('b', [dict([( 'd', [dict([('f', dict([('g', True), ('h', 7)]))])])])])])
     >>> visit_input_values(inputs, nested, visitor)
     name=a, prefix=, prefixed_name=a, prefixed_label=a, value=None
     No value found for 'a'.
@@ -274,7 +273,6 @@ def update_dataset_ids(input_values, translate_values, src):
 def populate_state(request_context, inputs, incoming, state, errors=None, context=None, check=True, simple_errors=True, input_format='legacy'):
     """
     Populates nested state dict from incoming parameter values.
-    >>> from collections import OrderedDict
     >>> from galaxy.util import XML
     >>> from galaxy.util.bunch import Bunch
     >>> from galaxy.tools.parameters.basic import TextToolParameter, BooleanToolParameter
@@ -294,15 +292,15 @@ def populate_state(request_context, inputs, incoming, state, errors=None, contex
     >>> h = TextToolParameter(None, XML('<param name="h"/>'))
     >>> i = TextToolParameter(None, XML('<param name="i"/>'))
     >>> b.name = 'b'
-    >>> b.inputs = OrderedDict([('c', c), ('d', d)])
+    >>> b.inputs = dict([('c', c), ('d', d)])
     >>> d.name = 'd'
-    >>> d.inputs = OrderedDict([('e', e), ('f', f)])
+    >>> d.inputs = dict([('e', e), ('f', f)])
     >>> f.test_param = g
     >>> f.name = 'f'
     >>> f.cases = [Bunch(value='true', inputs= { 'h': h }), Bunch(value='false', inputs= { 'i': i })]
-    >>> inputs = OrderedDict([('a',a),('b',b)])
-    >>> flat = OrderedDict([('a', 1), ('b_0|c', 2), ('b_0|d_0|e', 3), ('b_0|d_0|f|h', 4), ('b_0|d_0|f|g', True)])
-    >>> state = OrderedDict()
+    >>> inputs = dict([('a',a),('b',b)])
+    >>> flat = dict([('a', 1), ('b_0|c', 2), ('b_0|d_0|e', 3), ('b_0|d_0|f|h', 4), ('b_0|d_0|f|g', True)])
+    >>> state = {}
     >>> populate_state(trans, inputs, flat, state, check=False)
     >>> print(state['a'])
     1
@@ -314,7 +312,7 @@ def populate_state(request_context, inputs, incoming, state, errors=None, contex
     4
     >>> # now test with input_format='21.01'
     >>> nested = {'a': 1, 'b': [{'c': 2, 'd': [{'e': 3, 'f': {'h': 4, 'g': True}}]}]}
-    >>> state_new = OrderedDict()
+    >>> state_new = {}
     >>> populate_state(trans, inputs, nested, state_new, check=False, input_format='21.01')
     >>> print(state_new['a'])
     1
diff --git a/lib/galaxy/tools/parameters/meta.py b/lib/galaxy/tools/parameters/meta.py
index 7dc260df889..da6566ecc5c 100644
--- a/lib/galaxy/tools/parameters/meta.py
+++ b/lib/galaxy/tools/parameters/meta.py
@@ -1,7 +1,7 @@
 import copy
 import itertools
 import logging
-from collections import namedtuple, OrderedDict
+from collections import namedtuple
 
 from galaxy import (
     exceptions,
@@ -178,7 +178,7 @@ def expand_meta_parameters(trans, tool, incoming):
         if not incoming_key.startswith('__'):
             process_key(incoming_key, incoming_value=incoming_value, d=nested_dict)
 
-    reordered_incoming = OrderedDict()
+    reordered_incoming = {}
 
     def visitor(input, value, prefix, prefixed_name, prefixed_label, error, **kwargs):
         if prefixed_name in incoming_copy:
diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py
index 8d14362f9f7..729cac2f168 100644
--- a/lib/galaxy/tools/toolbox/base.py
+++ b/lib/galaxy/tools/toolbox/base.py
@@ -4,10 +4,7 @@ import os
 import string
 import time
 import urllib.request
-from collections import (
-    namedtuple,
-    OrderedDict
-)
+from collections import namedtuple
 from errno import ENOENT
 from urllib.parse import urlparse
 
@@ -103,7 +100,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
         # In-memory dictionary that defines the layout of the tool panel.
         self._tool_panel = ToolPanelElements()
         self._index = 0
-        self.data_manager_tools = OrderedDict()
+        self.data_manager_tools = {}
         self._lineage_map = LineageMap(app)
         # Sets self._integrated_tool_panel and self._integrated_tool_panel_config_has_contents
         self._init_integrated_tool_panel(app.config)
diff --git a/lib/galaxy/tools/wrappers.py b/lib/galaxy/tools/wrappers.py
index 5a9bf0f0b90..3427f77732b 100644
--- a/lib/galaxy/tools/wrappers.py
+++ b/lib/galaxy/tools/wrappers.py
@@ -1,7 +1,6 @@
 import logging
 import shlex
 import tempfile
-from collections import OrderedDict
 from functools import total_ordering
 
 from galaxy import exceptions
@@ -453,7 +452,7 @@ class DatasetCollectionWrapper(ToolParameterValueWrapper, HasDatasets):
         self.collection = collection
 
         elements = collection.elements
-        element_instances = OrderedDict()
+        element_instances = {}
 
         element_instance_list = []
         for dataset_collection_element in elements:
diff --git a/lib/galaxy/util/permutations.py b/lib/galaxy/util/permutations.py
index 577d00a57fe..8b3314580a4 100644
--- a/lib/galaxy/util/permutations.py
+++ b/lib/galaxy/util/permutations.py
@@ -6,7 +6,6 @@ first.
 Maybe this doesn't make sense and maybe much of this stuff could be replaced
 with itertools product and permutations. These are open questions.
 """
-from collections import OrderedDict
 
 from galaxy.exceptions import MessageException
 from galaxy.util.bunch import Bunch
@@ -42,9 +41,9 @@ def expand_multi_inputs(inputs, classifier, key_filter=None):
 def __split_inputs(inputs, classifier, key_filter):
     key_filter = key_filter or (lambda x: True)
 
-    single_inputs = OrderedDict()
-    matched_multi_inputs = OrderedDict()
-    multiplied_multi_inputs = OrderedDict()
+    single_inputs = {}
+    matched_multi_inputs = {}
+    multiplied_multi_inputs = {}
 
     for input_key in filter(key_filter, inputs):
         input_type, expanded_val = classifier(input_key)
diff --git a/lib/galaxy/util/simplegraph.py b/lib/galaxy/util/simplegraph.py
index 0ba0b958c0b..936e36a1936 100644
--- a/lib/galaxy/util/simplegraph.py
+++ b/lib/galaxy/util/simplegraph.py
@@ -3,8 +3,6 @@ Fencepost-simple graph structure implementation.
 """
 # Currently (2013.7.12) only used in easing the parsing of graph datatype data.
 
-from collections import OrderedDict
-
 
 class SimpleGraphNode:
     """
@@ -58,7 +56,7 @@ class SimpleGraph:
 
     def __init__(self, nodes=None, edges=None):
         # use an odict so that edge indeces actually match the final node list indeces
-        self.nodes = nodes or OrderedDict()
+        self.nodes = nodes or {}
         self.edges = edges or []
 
     def add_node(self, node_id, **data):
diff --git a/lib/galaxy/util/tool_shed/common_util.py b/lib/galaxy/util/tool_shed/common_util.py
index 846b121247d..4055923194a 100644
--- a/lib/galaxy/util/tool_shed/common_util.py
+++ b/lib/galaxy/util/tool_shed/common_util.py
@@ -2,7 +2,6 @@ import errno
 import json
 import logging
 import os
-from collections import OrderedDict
 from urllib.parse import urljoin
 
 from routes import url_for
@@ -36,16 +35,16 @@ def check_for_missing_tools(app, tool_panel_configs, latest_tool_migration_scrip
                                                        'migrate', 'scripts',
                                                        '%04d_tools.xml' % latest_tool_migration_script_number))
     # Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config.
-    migrated_tool_configs_dict = OrderedDict()
+    migrated_tool_configs_dict = {}
     tree, error_message = xml_util.parse_xml(tools_xml_file_path)
     if tree is None:
-        return False, OrderedDict()
+        return False, {}
     root = tree.getroot()
     tool_shed = root.get('name')
     tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, tool_shed)
     # The default behavior is that the tool shed is down.
     tool_shed_accessible = False
-    missing_tool_configs_dict = OrderedDict()
+    missing_tool_configs_dict = {}
     if tool_shed_url:
         for elem in root:
             if elem.tag == 'repository':
diff --git a/lib/galaxy/util/topsort.py b/lib/galaxy/util/topsort.py
index 691d2604695..a0bd490deac 100644
--- a/lib/galaxy/util/topsort.py
+++ b/lib/galaxy/util/topsort.py
@@ -36,7 +36,6 @@ then CycleError is raised, and the exception object supports
 many methods to help analyze and break the cycles.  This requires
 a good deal more code than topsort itself!
 """
-from collections import OrderedDict
 
 
 class CycleError(Exception):
@@ -88,7 +87,7 @@ class CycleError(Exception):
     def get_preds(self):
         if self.preds is not None:
             return self.preds
-        self.preds = preds = OrderedDict()
+        self.preds = preds = {}
         remaining_elts = self.get_elements()
         for x in remaining_elts:
             preds[x] = []
@@ -117,7 +116,7 @@ class CycleError(Exception):
         from random import choice
         x = choice(remaining_elts)
         answer = []
-        index = OrderedDict()
+        index = {}
         in_answer = index.has_key
         while not in_answer(x):
             index[x] = len(answer)  # index of x in answer
@@ -130,8 +129,8 @@ class CycleError(Exception):
 
 
 def _numpreds_and_successors_from_pairlist(pairlist):
-    numpreds = OrderedDict()   # elt -> # of predecessors
-    successors = OrderedDict()  # elt -> list of successors
+    numpreds = {}   # elt -> # of predecessors
+    successors = {}  # elt -> list of successors
     for first, second in pairlist:
         # make sure every elt is a key in numpreds
         if first not in numpreds:
diff --git a/lib/galaxy/util/yaml_util.py b/lib/galaxy/util/yaml_util.py
index 6d7e8ae880e..2fae2ed6936 100644
--- a/lib/galaxy/util/yaml_util.py
+++ b/lib/galaxy/util/yaml_util.py
@@ -29,7 +29,7 @@ class OrderedLoader(SafeLoader):
 def ordered_load(stream, merge_duplicate_keys=False):
     """
     Parse the first YAML document in a stream and produce the corresponding
-    Python object, using OrderedDicts instead of dicts.
+    Python object.
 
     If merge_duplicate_keys is True, merge the values of duplicate mapping keys
     into a list, as the uWSGI "dumb" YAML parser would do.
@@ -38,7 +38,7 @@ def ordered_load(stream, merge_duplicate_keys=False):
     """
     def construct_mapping(loader, node, deep=False):
         loader.flatten_mapping(node)
-        mapping = OrderedDict()
+        mapping = {}
         merged_duplicate = {}
         for key_node, value_node in node.value:
             key = loader.construct_object(key_node, deep=deep)
diff --git a/lib/galaxy/visualization/plugins/registry.py b/lib/galaxy/visualization/plugins/registry.py
index 12460ef24c1..47b337a43a7 100644
--- a/lib/galaxy/visualization/plugins/registry.py
+++ b/lib/galaxy/visualization/plugins/registry.py
@@ -7,7 +7,6 @@ Lower level of visualization framework which does three main things:
 import logging
 import os
 import weakref
-from collections import OrderedDict
 
 from galaxy.exceptions import ObjectNotFound
 from galaxy.util import (
@@ -66,7 +65,7 @@ class VisualizationsRegistry:
         self.additional_template_paths = []
         self.directories = []
         self.skip_bad_plugins = skip_bad_plugins
-        self.plugins = OrderedDict()
+        self.plugins = {}
         self.directories = config_directories_from_setting(directories_setting, app.config.root)
         self._load_configuration()
         self._load_plugins()
@@ -107,8 +106,6 @@ class VisualizationsRegistry:
         """
         Search ``self.directories`` for potential plugins, load them, and cache
         in ``self.plugins``.
-        :rtype:                 OrderedDict
-        :returns:               ``self.plugins``
         """
         for plugin_path in self._find_plugins():
             try:
diff --git a/lib/galaxy/web/framework/helpers/grids.py b/lib/galaxy/web/framework/helpers/grids.py
index 31db02a9262..61725c205cc 100644
--- a/lib/galaxy/web/framework/helpers/grids.py
+++ b/lib/galaxy/web/framework/helpers/grids.py
@@ -1,6 +1,5 @@
 import logging
 import math
-from collections import OrderedDict
 from json import dumps, loads
 from typing import Dict, List, Optional
 
@@ -492,7 +491,7 @@ class SharingStatusColumn(GridColumn):
 
     def get_accepted_filters(self):
         """ Returns a list of accepted filters for this column. """
-        accepted_filter_labels_and_vals = OrderedDict()
+        accepted_filter_labels_and_vals = {}
         accepted_filter_labels_and_vals["private"] = "private"
         accepted_filter_labels_and_vals["shared"] = "shared"
         accepted_filter_labels_and_vals["accessible"] = "accessible"
diff --git a/lib/galaxy/web/legacy_framework/grids.py b/lib/galaxy/web/legacy_framework/grids.py
index 13b03171e35..f1bf7c1232c 100644
--- a/lib/galaxy/web/legacy_framework/grids.py
+++ b/lib/galaxy/web/legacy_framework/grids.py
@@ -1,6 +1,5 @@
 import logging
 import math
-from collections import OrderedDict
 from json import dumps, loads
 from typing import Dict, List, Optional
 
@@ -457,7 +456,7 @@ class SharingStatusColumn(GridColumn):
 
     def get_accepted_filters(self):
         """ Returns a list of accepted filters for this column. """
-        accepted_filter_labels_and_vals = OrderedDict()
+        accepted_filter_labels_and_vals = {}
         accepted_filter_labels_and_vals["private"] = "private"
         accepted_filter_labels_and_vals["shared"] = "shared"
         accepted_filter_labels_and_vals["accessible"] = "accessible"
diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py
index 8dd4a781cd4..c737d062ded 100644
--- a/lib/galaxy/webapps/galaxy/api/users.py
+++ b/lib/galaxy/webapps/galaxy/api/users.py
@@ -5,7 +5,6 @@ import copy
 import json
 import logging
 import re
-from collections import OrderedDict
 
 from markupsafe import escape
 from sqlalchemy import (
@@ -710,9 +709,9 @@ class UserAPIController(BaseAPIController, UsesTagsMixin, BaseUIController, Uses
             inputs.append({'type': 'section', 'title': filter_title, 'name': filter_type, 'expanded': True, 'inputs': filter_inputs})
 
     def _get_filter_types(self, trans):
-        return OrderedDict([('toolbox_tool_filters', {'title': 'Tools', 'config': trans.app.config.user_tool_filters}),
-                            ('toolbox_section_filters', {'title': 'Sections', 'config': trans.app.config.user_tool_section_filters}),
-                            ('toolbox_label_filters', {'title': 'Labels', 'config': trans.app.config.user_tool_label_filters})])
+        return {'toolbox_tool_filters': {'title': 'Tools', 'config': trans.app.config.user_tool_filters},
+                'toolbox_section_filters': {'title': 'Sections', 'config': trans.app.config.user_tool_section_filters},
+                'toolbox_label_filters': {'title': 'Labels', 'config': trans.app.config.user_tool_label_filters}}
 
     @expose_api
     def api_key(self, trans, id, payload=None, **kwd):
diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py
index 36b1e38e098..a2ed176c1ec 100644
--- a/lib/galaxy/webapps/galaxy/controllers/admin.py
+++ b/lib/galaxy/webapps/galaxy/controllers/admin.py
@@ -1,7 +1,6 @@
 import imp
 import logging
 import os
-from collections import OrderedDict
 from datetime import datetime, timedelta
 
 from sqlalchemy import and_, false, or_
@@ -882,7 +881,7 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP
     def review_tool_migration_stages(self, trans, **kwd):
         message = escape(util.restore_text(kwd.get('message', '')))
         status = util.restore_text(kwd.get('status', 'done'))
-        migration_stages_dict = OrderedDict()
+        migration_stages_dict = {}
         # FIXME: this isn't valid in an installed context
         migration_scripts_dir = os.path.abspath(os.path.join(trans.app.config.root, 'lib', 'galaxy', 'tool_shed', 'galaxy_install', 'migrate', 'versions'))
         modules = os.listdir(migration_scripts_dir)
diff --git a/lib/galaxy/webapps/galaxy/controllers/history.py b/lib/galaxy/webapps/galaxy/controllers/history.py
index 1adc7bbe97b..4a1c1d7e0b9 100644
--- a/lib/galaxy/webapps/galaxy/controllers/history.py
+++ b/lib/galaxy/webapps/galaxy/controllers/history.py
@@ -1,5 +1,4 @@
 import logging
-from collections import OrderedDict
 
 from markupsafe import escape
 from sqlalchemy import (
@@ -485,7 +484,7 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt
         items = []
         # First go through and group hdas by job, if there is no job they get
         # added directly to items
-        jobs = OrderedDict()
+        jobs = {}
         for hda in history.active_datasets:
             if hda.visible is False:
                 continue
@@ -509,7 +508,7 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt
                 else:
                     jobs[job] = [(hda, None)]
         # Second, go through the jobs and connect to workflows
-        wf_invocations = OrderedDict()
+        wf_invocations = {}
         for job, hdas in jobs.items():
             # Job is attached to a workflow step, follow it to the
             # workflow_invocation and group
diff --git a/lib/galaxy/webapps/reports/controllers/history.py b/lib/galaxy/webapps/reports/controllers/history.py
index c921821657c..f5ff791061c 100644
--- a/lib/galaxy/webapps/reports/controllers/history.py
+++ b/lib/galaxy/webapps/reports/controllers/history.py
@@ -1,4 +1,3 @@
-import collections
 import logging
 
 import sqlalchemy as sa
@@ -111,7 +110,7 @@ class History(BaseUIController):
             users = users[:user_cutoff]
 
         # to keep ordered
-        data = collections.OrderedDict()
+        data = {}
         for user in users:
             dataset = datasets.get(user, [0, 0])
             history = histories.get(user, 0)
@@ -172,8 +171,7 @@ class History(BaseUIController):
         possible_status = {"ok": 0, "upload": 1, "paused": 2, "queued": 3, "error": 4, "discarded": 5}
         number_of_possible_status = len(possible_status) + 1  # + 1 to handle unknown status!
 
-        # to keep ordered
-        datas = collections.OrderedDict()
+        datas = {}
         for no, name in enumerate(names):
             if name not in datas:
                 if user_cutoff > 0:
diff --git a/lib/galaxy/webapps/reports/controllers/tools.py b/lib/galaxy/webapps/reports/controllers/tools.py
index 69b8863a6b9..b18cd351049 100644
--- a/lib/galaxy/webapps/reports/controllers/tools.py
+++ b/lib/galaxy/webapps/reports/controllers/tools.py
@@ -1,4 +1,3 @@
-import collections
 import logging
 from datetime import timedelta
 
@@ -85,7 +84,7 @@ class Tools(BaseUIController):
             lambda v: tools_and_jobs_ok.get(v, 0),
             lambda v: tools_and_jobs_error.get(v, 0))
 
-        data = collections.OrderedDict()
+        data = {}
 
         # select count(id), tool_id from job where state='ok' group by tool_id;
         tools_and_jobs_ok = sa.select((galaxy.model.Job.table.c.tool_id .label('tool'),
@@ -139,7 +138,7 @@ class Tools(BaseUIController):
         if tool is None:
             raise TypeError("Tool can't be None")
 
-        data = collections.OrderedDict()
+        data = {}
 
         # select count(id), create_time from job where state='ok' and tool_id=$tool group by date;
         date_and_jobs_ok = sa.select((sa.func.date(galaxy.model.Job.table.c.create_time).label('date'),
@@ -194,7 +193,7 @@ class Tools(BaseUIController):
         color = True if kwd.get("color", '') == "True" else False
 
         data = {}
-        ordered_data = collections.OrderedDict()
+        ordered_data = {}
 
         sort_keys = (
             lambda v: v.lower(),
@@ -260,7 +259,7 @@ class Tools(BaseUIController):
         if tool is None:
             raise ValueError("Tool can't be None")
 
-        ordered_data = collections.OrderedDict()
+        ordered_data = {}
         sort_keys = [(lambda v, i=i: v[i]) for i in range(4)]
 
         jobs_times = sa.select((sa.func.date_trunc('month', galaxy.model.Job.table.c.create_time).label('date'),
@@ -314,7 +313,7 @@ class Tools(BaseUIController):
             else:
                 counter[error[0]] = [1, error[1]]
 
-        data = collections.OrderedDict()
+        data = {}
         keys = list(counter.keys())
         if cutoff:
             keys = keys[:cutoff]
diff --git a/lib/galaxy/workflow/extract.py b/lib/galaxy/workflow/extract.py
index a6c3354513d..e3a1e3e28d6 100644
--- a/lib/galaxy/workflow/extract.py
+++ b/lib/galaxy/workflow/extract.py
@@ -2,7 +2,6 @@
 histories.
 """
 import logging
-from collections import OrderedDict
 
 from galaxy import exceptions, model
 from galaxy.tool_util.parser import ToolOutputCollectionPart
@@ -196,7 +195,7 @@ class WorkflowSummary:
             history = trans.get_history()
         self.history = history
         self.warnings = set()
-        self.jobs = OrderedDict()
+        self.jobs = {}
         self.job_id2representative_job = {}  # map a non-fake job id to its representative job
         self.implicit_map_jobs = []
         self.collection_types = {}
diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py
index ca526f45008..ef259d9cd73 100644
--- a/lib/galaxy/workflow/modules.py
+++ b/lib/galaxy/workflow/modules.py
@@ -4,7 +4,7 @@ Modules used in building workflows
 import json
 import logging
 import re
-from collections import defaultdict, OrderedDict
+from collections import defaultdict
 
 import packaging.version
 
@@ -707,7 +707,7 @@ class InputDataModule(InputModule):
     def get_inputs(self):
         parameter_def = self._parse_state_into_dict()
         optional = parameter_def["optional"]
-        inputs = OrderedDict()
+        inputs = {}
         inputs["optional"] = optional_param(optional)
         inputs["format"] = format_param(self.trans, parameter_def.get("format"))
         return inputs
@@ -730,7 +730,7 @@ class InputDataCollectionModule(InputModule):
             {"value": "list:paired", "label": "List of Dataset Pairs"},
         ]
         input_collection_type = TextToolParameter(None, collection_type_source)
-        inputs = OrderedDict()
+        inputs = {}
         inputs["collection_type"] = input_collection_type
         inputs["optional"] = optional_param(optional)
         inputs["format"] = format_param(self.trans, parameter_def.get("format"))
@@ -859,7 +859,7 @@ class InputParameterModule(WorkflowModule):
 
             when_this_type = ConditionalWhen()
             when_this_type.value = param_type
-            when_this_type.inputs = OrderedDict()
+            when_this_type.inputs = {}
             when_this_type.inputs["optional"] = optional_cond
 
             specify_default_checked = "default" in parameter_def
@@ -871,24 +871,24 @@ class InputParameterModule(WorkflowModule):
 
             when_specify_default_true = ConditionalWhen()
             when_specify_default_true.value = "true"
-            when_specify_default_true.inputs = OrderedDict()
+            when_specify_default_true.inputs = {}
             when_specify_default_true.inputs["default"] = input_default_value
 
             when_specify_default_false = ConditionalWhen()
             when_specify_default_false.value = "false"
-            when_specify_default_false.inputs = OrderedDict()
+            when_specify_default_false.inputs = {}
 
             specify_default_cond_cases = [when_specify_default_true, when_specify_default_false]
             specify_default_cond.cases = specify_default_cond_cases
 
             when_true = ConditionalWhen()
             when_true.value = "true"
-            when_true.inputs = OrderedDict()
+            when_true.inputs = {}
             when_true.inputs["default"] = specify_default_cond
 
             when_false = ConditionalWhen()
             when_false.value = "false"
-            when_false.inputs = OrderedDict()
+            when_false.inputs = {}
 
             optional_cases = [when_true, when_false]
             optional_cond.cases = optional_cases
@@ -916,19 +916,19 @@ class InputParameterModule(WorkflowModule):
 
                 when_restrict_none = ConditionalWhen()
                 when_restrict_none.value = "none"
-                when_restrict_none.inputs = OrderedDict()
+                when_restrict_none.inputs = {}
 
                 when_restrict_connections = ConditionalWhen()
                 when_restrict_connections.value = "onConnections"
-                when_restrict_connections.inputs = OrderedDict()
+                when_restrict_connections.inputs = {}
 
                 when_restrict_static_restrictions = ConditionalWhen()
                 when_restrict_static_restrictions.value = "staticRestrictions"
-                when_restrict_static_restrictions.inputs = OrderedDict()
+                when_restrict_static_restrictions.inputs = {}
 
                 when_restrict_static_suggestions = ConditionalWhen()
                 when_restrict_static_suggestions.value = "staticSuggestions"
-                when_restrict_static_suggestions.inputs = OrderedDict()
+                when_restrict_static_suggestions.inputs = {}
 
                 # Repeats don't work - so use common separated list for now.
 
@@ -953,7 +953,7 @@ class InputParameterModule(WorkflowModule):
             cases.append(when_this_type)
 
         parameter_type_cond.cases = cases
-        return OrderedDict([("parameter_definition", parameter_type_cond)])
+        return {"parameter_definition": parameter_type_cond}
 
     def get_runtime_inputs(self, connections=None, **kwds):
         parameter_def = self._parse_state_into_dict()
diff --git a/lib/galaxy/workflow/run.py b/lib/galaxy/workflow/run.py
index 28acc42b76c..c261f23e7e2 100644
--- a/lib/galaxy/workflow/run.py
+++ b/lib/galaxy/workflow/run.py
@@ -1,6 +1,5 @@
 import logging
 import uuid
-from collections import OrderedDict
 
 from galaxy import model
 from galaxy.util import ExecutionTimer
@@ -274,7 +273,7 @@ STEP_OUTPUT_DELAYED = object()
 class WorkflowProgress:
 
     def __init__(self, workflow_invocation, inputs_by_step_id, module_injector, param_map, jobs_per_scheduling_iteration=-1):
-        self.outputs = OrderedDict()
+        self.outputs = {}
         self.module_injector = module_injector
         self.workflow_invocation = workflow_invocation
         self.inputs_by_step_id = inputs_by_step_id
diff --git a/lib/tool_shed/dependencies/attribute_handlers.py b/lib/tool_shed/dependencies/attribute_handlers.py
index 9ab1e805b9d..a8deaaf6f60 100644
--- a/lib/tool_shed/dependencies/attribute_handlers.py
+++ b/lib/tool_shed/dependencies/attribute_handlers.py
@@ -1,6 +1,5 @@
 import copy
 import logging
-from collections import OrderedDict
 
 from galaxy.util import asbool
 from galaxy.web import url_for
@@ -71,8 +70,8 @@ class RepositoryDependencyAttributeHandler:
         if len(sub_elems) > 0:
             # At this point, a <repository> tag will point only to a package.
             # <package name="xorg_macros" version="1.17.1" />
-            # Coerce the list to an OrderedDict().
-            sub_elements = OrderedDict()
+            # Coerce the list to dict.
+            sub_elements = {}
             packages = []
             for sub_elem in sub_elems:
                 sub_elem_type = sub_elem.tag
@@ -88,7 +87,7 @@ class RepositoryDependencyAttributeHandler:
             # We're exporting the repository, so eliminate all toolshed and changeset_revision attributes
             # from the <repository> tag.
             if toolshed or changeset_revision:
-                attributes = OrderedDict()
+                attributes = {}
                 attributes['name'] = name
                 attributes['owner'] = owner
                 prior_installation_required = elem.get('prior_installation_required')
diff --git a/lib/tool_shed/repository_types/registry.py b/lib/tool_shed/repository_types/registry.py
index 0efd339086c..83d89eab177 100644
--- a/lib/tool_shed/repository_types/registry.py
+++ b/lib/tool_shed/repository_types/registry.py
@@ -1,5 +1,4 @@
 import logging
-from collections import OrderedDict
 
 from . import (
     repository_suite_definition,
@@ -13,7 +12,7 @@ log = logging.getLogger(__name__)
 class Registry:
 
     def __init__(self):
-        self.repository_types_by_label = OrderedDict()
+        self.repository_types_by_label = {}
         self.repository_types_by_label['unrestricted'] = unrestricted.Unrestricted()
         self.repository_types_by_label['repository_suite_definition'] = repository_suite_definition.RepositorySuiteDefinition()
         self.repository_types_by_label['tool_dependency_definition'] = tool_dependency_definition.ToolDependencyDefinition()
diff --git a/lib/tool_shed/util/review_util.py b/lib/tool_shed/util/review_util.py
index 876c34f7929..0e1f5980ce6 100644
--- a/lib/tool_shed/util/review_util.py
+++ b/lib/tool_shed/util/review_util.py
@@ -1,5 +1,4 @@
 import logging
-from collections import OrderedDict
 
 from sqlalchemy import and_
 
@@ -74,7 +73,7 @@ def get_previous_repository_reviews(app, repository, changeset_revision):
     """
     repo = repository.hg_repo
     reviewed_revision_hashes = [review.changeset_revision for review in repository.reviews]
-    previous_reviews_dict = OrderedDict()
+    previous_reviews_dict = {}
     for changeset in hg_util.reversed_upper_bounded_changelog(repo, changeset_revision):
         previous_changeset_revision = str(repo[changeset])
         if previous_changeset_revision in reviewed_revision_hashes:
diff --git a/lib/tool_shed/webapp/controllers/repository_review.py b/lib/tool_shed/webapp/controllers/repository_review.py
index 975571fcb54..2c22865f997 100644
--- a/lib/tool_shed/webapp/controllers/repository_review.py
+++ b/lib/tool_shed/webapp/controllers/repository_review.py
@@ -1,5 +1,4 @@
 import logging
-from collections import OrderedDict
 
 from sqlalchemy import (
     and_,
@@ -240,7 +239,7 @@ class RepositoryReviewController(BaseUIController, ratings_util.ItemRatings):
         status = kwd.get('status', 'done')
         review_id = kwd.get('id', None)
         review = review_util.get_review(trans.app, review_id)
-        components_dict = OrderedDict()
+        components_dict = {}
         for component in review_util.get_components(trans.app):
             components_dict[component.name] = dict(component=component, component_review=None)
         repository = review.repository
@@ -487,7 +486,7 @@ class RepositoryReviewController(BaseUIController, ratings_util.ItemRatings):
             repo = repository.hg_repo
             metadata_revision_hashes = [metadata_revision.changeset_revision for metadata_revision in repository.metadata_revisions]
             reviewed_revision_hashes = [review.changeset_revision for review in repository.reviews]
-            reviews_dict = OrderedDict()
+            reviews_dict = {}
             for changeset in hg_util.get_reversed_changelog_changesets(repo):
                 changeset_revision = str(repo[changeset])
                 if changeset_revision in metadata_revision_hashes or changeset_revision in reviewed_revision_hashes:

From 7d0ec29fa06ae6f04c0ae6fc193a390bc11cddcd Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Thu, 4 Feb 2021 12:48:02 +0100
Subject: [PATCH 12/32] Drop legacy metadata mode

And always serialize input dataset metadata using the model exports.
This means we don't have to pickle anymore, which breaks with weakrefs
used by sqlalchemy-json's Mutable class.
---
 doc/source/admin/galaxy_options.rst         |  14 +-
 lib/galaxy/config/sample/galaxy.yml.sample  |   6 +-
 lib/galaxy/metadata/__init__.py             | 239 ++------------------
 lib/galaxy/metadata/set_metadata.py         |  81 +------
 lib/galaxy/webapps/galaxy/config_schema.yml |   2 +-
 test/unit/tools/test_metadata.py            |  12 -
 6 files changed, 33 insertions(+), 321 deletions(-)

diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst
index 6469d27017f..86479656e0d 100644
--- a/doc/source/admin/galaxy_options.rst
+++ b/doc/source/admin/galaxy_options.rst
@@ -3851,13 +3851,13 @@
 ~~~~~~~~~~~~~~~~~~~~~
 
 :Description:
-    Determines how metadata will be set. Valid values are `directory`,
-    `extended` and `legacy`. In extended mode jobs will decide if a
-    tool run failed, the object stores configuration is serialized and
-    made available to the job and is used for writing output datasets
-    to the object store as part of the job and dynamic output
-    discovery (e.g. discovered datasets <discover_datasets>,
-    unpopulated collections, etc) happens as part of the job.
+    Determines how metadata will be set. Valid values are `directory`
+    and `extended`. In extended mode jobs will decide if a tool run
+    failed, the object stores configuration is serialized and made
+    available to the job and is used for writing output datasets to
+    the object store as part of the job and dynamic output discovery
+    (e.g. discovered datasets <discover_datasets>, unpopulated
+    collections, etc) happens as part of the job.
 :Default: ``directory``
 :Type: str
 
diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample
index 7fccb510097..9bf13d71da0 100644
--- a/lib/galaxy/config/sample/galaxy.yml.sample
+++ b/lib/galaxy/config/sample/galaxy.yml.sample
@@ -1902,9 +1902,9 @@ galaxy:
   # database.
   #enable_job_recovery: true
 
-  # Determines how metadata will be set. Valid values are `directory`,
-  # `extended` and `legacy`. In extended mode jobs will decide if a tool
-  # run failed, the object stores configuration is serialized and made
+  # Determines how metadata will be set. Valid values are `directory`
+  # and `extended`. In extended mode jobs will decide if a tool run
+  # failed, the object stores configuration is serialized and made
   # available to the job and is used for writing output datasets to the
   # object store as part of the job and dynamic output discovery (e.g.
   # discovered datasets <discover_datasets>, unpopulated collections,
diff --git a/lib/galaxy/metadata/__init__.py b/lib/galaxy/metadata/__init__.py
index bff6ac4baf1..f072015bd55 100644
--- a/lib/galaxy/metadata/__init__.py
+++ b/lib/galaxy/metadata/__init__.py
@@ -3,17 +3,14 @@
 import abc
 import json
 import os
-import pickle
 import shutil
-import tempfile
 from logging import getLogger
-from os.path import abspath
 
 import galaxy.model
 from galaxy.model import store
 from galaxy.model.metadata import FileParameter, MetadataTempFile
 from galaxy.model.store import DirectoryModelExportStore
-from galaxy.util import in_directory, safe_makedirs
+from galaxy.util import safe_makedirs
 
 log = getLogger(__name__)
 
@@ -23,7 +20,7 @@ SET_METADATA_SCRIPT = 'from galaxy_ext.metadata.set_metadata import set_metadata
 def get_metadata_compute_strategy(config, job_id, metadata_strategy_override=None, tool_id=None):
     metadata_strategy = metadata_strategy_override or config.metadata_strategy
     if metadata_strategy == "legacy":
-        return JobExternalOutputMetadataWrapper(job_id)
+        raise Exception('legacy metadata_strategy has been removed')
     elif metadata_strategy == "extended" and tool_id != "__SET_METADATA__":
         return ExtendedDirectoryMetadataGenerator(job_id)
     else:
@@ -130,17 +127,15 @@ class PortableDirectoryMetadataGenerator(MetadataCollectionStrategy):
         outputs = {}
         output_collections = {}
 
-        real_metadata_object = self.write_object_store_conf
         for name, dataset in datasets_dict.items():
             assert name is not None
             assert name not in outputs
-
             key = name
 
             def _metadata_path(what):
                 return os.path.join(metadata_dir, f"metadata_{what}_{key}")
 
-            _initialize_metadata_inputs(dataset, _metadata_path, tmp_dir, kwds, real_metadata_object=real_metadata_object)
+            _initialize_metadata_inputs(dataset, _metadata_path, tmp_dir, kwds, real_metadata_object=self.write_object_store_conf)
 
             outputs[name] = {
                 "filename_override": _get_filename_override(output_fnames, dataset.file_name),
@@ -159,19 +154,19 @@ class PortableDirectoryMetadataGenerator(MetadataCollectionStrategy):
             "outputs": outputs,
         }
 
+        # export model objects and object store configuration for extended metadata also.
+        export_directory = os.path.join(metadata_dir, "outputs_new")
+        with DirectoryModelExportStore(export_directory, for_edit=True, serialize_dataset_objects=True) as export_store:
+            for dataset in datasets_dict.values():
+                export_store.add_dataset(dataset)
+
+            for name, dataset_collection in out_collections.items():
+                export_store.add_dataset_collection(dataset_collection)
+                output_collections[name] = {
+                    'id': dataset_collection.id,
+                }
+
         if self.write_object_store_conf:
-            # export model objects and object store configuration for extended metadata also.
-            export_directory = os.path.join(metadata_dir, "outputs_new")
-            with DirectoryModelExportStore(export_directory, for_edit=True, serialize_dataset_objects=True) as export_store:
-                for dataset in datasets_dict.values():
-                    export_store.add_dataset(dataset)
-
-                for name, dataset_collection in out_collections.items():
-                    export_store.add_dataset_collection(dataset_collection)
-                    output_collections[name] = {
-                        'id': dataset_collection.id,
-                    }
-
             with open(os.path.join(metadata_dir, "object_store_conf.json"), "w") as f:
                 json.dump(object_store_conf, f)
 
@@ -241,195 +236,12 @@ class ExtendedDirectoryMetadataGenerator(PortableDirectoryMetadataGenerator):
         return dataset
 
 
-class JobExternalOutputMetadataWrapper(MetadataCollectionStrategy):
-    """
-    Class with methods allowing set_meta() to be called externally to the
-    Galaxy head.
-    This class allows access to external metadata filenames for all outputs
-    associated with a job.
-    We will use JSON as the medium of exchange of information, except for the
-    DatasetInstance object which will use pickle (in the future this could be
-    JSONified as well)
-    """
-    portable = False
-
-    def __init__(self, job_id):
-        self.job_id = job_id
-
-    def _get_output_filenames_by_dataset(self, dataset, sa_session):
-        if isinstance(dataset, galaxy.model.HistoryDatasetAssociation):
-            return sa_session.query(galaxy.model.JobExternalOutputMetadata) \
-                             .filter_by(job_id=self.job_id,
-                                        history_dataset_association_id=dataset.id,
-                                        is_valid=True) \
-                             .first()  # there should only be one or None
-        elif isinstance(dataset, galaxy.model.LibraryDatasetDatasetAssociation):
-            return sa_session.query(galaxy.model.JobExternalOutputMetadata) \
-                             .filter_by(job_id=self.job_id,
-                                        library_dataset_dataset_association_id=dataset.id,
-                                        is_valid=True) \
-                             .first()  # there should only be one or None
-        return None
-
-    def _get_dataset_metadata_key(self, dataset):
-        # Set meta can be called on library items and history items,
-        # need to make different keys for them, since ids can overlap
-        return "%s_%d" % (dataset.__class__.__name__, dataset.id)
-
-    def invalidate_external_metadata(self, datasets, sa_session):
-        for dataset in datasets:
-            jeom = self._get_output_filenames_by_dataset(dataset, sa_session)
-            # shouldn't be more than one valid, but you never know
-            while jeom:
-                jeom.is_valid = False
-                sa_session.add(jeom)
-                sa_session.flush()
-                jeom = self._get_output_filenames_by_dataset(dataset, sa_session)
-
-    def setup_external_metadata(self, datasets_dict, out_collections, sa_session, exec_dir=None,
-                                tmp_dir=None, dataset_files_path=None,
-                                output_fnames=None, config_root=None, use_bin=False,
-                                config_file=None, datatypes_config=None,
-                                job_metadata=None, provided_metadata_style=None, compute_tmp_dir=None,
-                                include_command=True, max_metadata_value_size=0,
-                                validate_outputs=False,
-                                object_store_conf=None, tool=None, job=None,
-                                kwds=None):
-        kwds = kwds or {}
-        if not job:
-            job = sa_session.query(galaxy.model.Job).get(self.job_id)
-        tmp_dir = _init_tmp_dir(tmp_dir)
-        _assert_datatypes_config(datatypes_config)
-
-        # path is calculated for Galaxy, may be different on compute - rewrite
-        # for the compute server.
-        def metadata_path_on_compute(path):
-            compute_path = path
-            if compute_tmp_dir and tmp_dir and in_directory(path, tmp_dir):
-                path_relative = os.path.relpath(path, tmp_dir)
-                compute_path = os.path.join(compute_tmp_dir, path_relative)
-            return compute_path
-
-        # fill in metadata_files_dict and return the command with args required to set metadata
-        def __metadata_files_list_to_cmd_line(metadata_files):
-            line = '"{},{},{},{},{},{}"'.format(
-                metadata_path_on_compute(metadata_files.filename_in),
-                metadata_path_on_compute(metadata_files.filename_kwds),
-                metadata_path_on_compute(metadata_files.filename_out),
-                metadata_path_on_compute(metadata_files.filename_results_code),
-                _get_filename_override(output_fnames, metadata_files.dataset.file_name),
-                metadata_path_on_compute(metadata_files.filename_override_metadata),
-            )
-            return line
-
-        datasets = list(datasets_dict.values())
-        if exec_dir is None:
-            exec_dir = os.path.abspath(os.getcwd())
-        if dataset_files_path is None:
-            dataset_files_path = galaxy.model.Dataset.file_path
-        if config_root is None:
-            config_root = os.path.abspath(os.getcwd())
-        metadata_files_list = []
-        for dataset in datasets:
-            key = self._get_dataset_metadata_key(dataset)
-            # future note:
-            # wonkiness in job execution causes build command line to be called more than once
-            # when setting metadata externally, via 'auto-detect' button in edit attributes, etc.,
-            # we don't want to overwrite (losing the ability to cleanup) our existing dataset keys and files,
-            # so we will only populate the dictionary once
-            metadata_files = self._get_output_filenames_by_dataset(dataset, sa_session)
-            if not metadata_files:
-                metadata_files = galaxy.model.JobExternalOutputMetadata(job=job, dataset=dataset)
-                # we are using tempfile to create unique filenames, tempfile always returns an absolute path
-                # we will use pathnames relative to the galaxy root, to accommodate instances where the galaxy root
-                # is located differently, i.e. on a cluster node with a different filesystem structure
-
-                def _metadata_path(what):
-                    return abspath(tempfile.NamedTemporaryFile(dir=tmp_dir, prefix=f"metadata_{what}_{key}_").name)
-
-                filename_in, filename_out, filename_results_code, filename_kwds, filename_override_metadata = _initialize_metadata_inputs(dataset, _metadata_path, tmp_dir, kwds)
-
-                # file to store existing dataset
-                metadata_files.filename_in = filename_in
-
-                # file to store metadata results of set_meta()
-                metadata_files.filename_out = filename_out
-
-                # file to store a 'return code' indicating the results of the set_meta() call
-                # results code is like (True/False - if setting metadata was successful/failed , exception or string of reason of success/failure )
-                metadata_files.filename_results_code = filename_results_code
-
-                # file to store kwds passed to set_meta()
-                metadata_files.filename_kwds = filename_kwds
-
-                # existing metadata file parameters need to be overridden with cluster-writable file locations
-                metadata_files.filename_override_metadata = filename_override_metadata
-
-                # add to session and flush
-                sa_session.add(metadata_files)
-                sa_session.flush()
-            metadata_files_list.append(metadata_files)
-        args = '"{}" "{}" {} {}'.format(metadata_path_on_compute(datatypes_config),
-                                    job_metadata,
-                                    " ".join(map(__metadata_files_list_to_cmd_line, metadata_files_list)),
-                                    max_metadata_value_size)
-        assert not use_bin
-        if include_command:
-            # return command required to build
-            with tempfile.NamedTemporaryFile(mode='w', suffix='.py', dir=tmp_dir, prefix="set_metadata_", delete=False) as temp:
-                temp.write(SET_METADATA_SCRIPT)
-            return 'python "{}" {}'.format(metadata_path_on_compute(temp.name), args)
-        else:
-            # return args to galaxy_ext.metadata.set_metadata required to build
-            return args
-
-    def external_metadata_set_successfully(self, dataset, name, sa_session, working_directory):
-        metadata_files = self._get_output_filenames_by_dataset(dataset, sa_session)
-        if not metadata_files:
-            return False  # this file doesn't exist
-        return self._metadata_results_from_file(dataset, metadata_files.filename_results_code)
-
-    def cleanup_external_metadata(self, sa_session):
-        log.debug('Cleaning up external metadata files')
-        for metadata_files in sa_session.query(galaxy.model.Job).get(self.job_id).external_output_metadata:
-            # we need to confirm that any MetadataTempFile files were removed, if not we need to remove them
-            # can occur if the job was stopped before completion, but a MetadataTempFile is used in the set_meta
-            MetadataTempFile.cleanup_from_JSON_dict_filename(metadata_files.filename_out)
-            dataset_key = self._get_dataset_metadata_key(metadata_files.dataset)
-            for key, fname in [('filename_in', metadata_files.filename_in),
-                               ('filename_out', metadata_files.filename_out),
-                               ('filename_results_code', metadata_files.filename_results_code),
-                               ('filename_kwds', metadata_files.filename_kwds),
-                               ('filename_override_metadata', metadata_files.filename_override_metadata)]:
-                try:
-                    os.remove(fname)
-                except Exception as e:
-                    log.debug(f'Failed to cleanup external metadata file ({key}) for {dataset_key}: {e}')
-
-    def set_job_runner_external_pid(self, pid, sa_session):
-        for metadata_files in sa_session.query(galaxy.model.Job).get(self.job_id).external_output_metadata:
-            metadata_files.job_runner_external_pid = pid
-            sa_session.add(metadata_files)
-            sa_session.flush()
-
-    def load_metadata(self, dataset, name, sa_session, working_directory, remote_metadata_directory=None):
-        # load metadata from file
-        # we need to no longer allow metadata to be edited while the job is still running,
-        # since if it is edited, the metadata changed on the running output will no longer match
-        # the metadata that was stored to disk for use via the external process,
-        # and the changes made by the user will be lost, without warning or notice
-        output_filename = self._get_output_filenames_by_dataset(dataset, sa_session).filename_out
-        self._load_metadata_from_path(dataset, output_filename, working_directory, remote_metadata_directory)
-
-
 def _initialize_metadata_inputs(dataset, path_for_part, tmp_dir, kwds, real_metadata_object=True):
-    filename_in = path_for_part("in")
     filename_out = path_for_part("out")
     filename_results_code = path_for_part("results")
     filename_kwds = path_for_part("kwds")
     filename_override_metadata = path_for_part("override")
 
-    _dump_dataset_instance_to(dataset, filename_in)
     open(filename_out, 'wt+')  # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible)
     # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible)
     json.dump((False, 'External set_meta() not called'), open(filename_results_code, 'wt+'))
@@ -446,26 +258,7 @@ def _initialize_metadata_inputs(dataset, path_for_part, tmp_dir, kwds, real_meta
 
     json.dump(override_metadata, open(filename_override_metadata, 'wt+'))
 
-    return filename_in, filename_out, filename_results_code, filename_kwds, filename_override_metadata
-
-
-def _assert_datatypes_config(datatypes_config):
-    if datatypes_config is None:
-        raise Exception('In setup_external_metadata, the received datatypes_config is None.')
-
-
-def _dump_dataset_instance_to(dataset_instance, file_path):
-    # FIXME: HACK
-    # sqlalchemy introduced 'expire_on_commit' flag for sessionmaker at version 0.5x
-    # This may be causing the dataset attribute of the dataset_association object to no-longer be loaded into memory when needed for pickling.
-    # For now, we'll simply 'touch' dataset_association.dataset to force it back into memory.
-    dataset_instance.dataset  # force dataset_association.dataset to be loaded before pickling
-    # A better fix could be setting 'expire_on_commit=False' on the session, or modifying where commits occur, or ?
-
-    # Touch also deferred column
-    dataset_instance._metadata
-
-    pickle.dump(dataset_instance, open(file_path, 'wb+'))
+    return filename_out, filename_results_code, filename_kwds, filename_override_metadata
 
 
 def _get_filename_override(output_fnames, file_name):
diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py
index 9fb3769e395..e47adfaafc0 100644
--- a/lib/galaxy/metadata/set_metadata.py
+++ b/lib/galaxy/metadata/set_metadata.py
@@ -13,7 +13,6 @@ constructed automatically).
 import json
 import logging
 import os
-import pickle
 import sys
 import traceback
 
@@ -23,10 +22,7 @@ import galaxy.model.mapping  # need to load this before we unpickle, in order to
 from galaxy.model import store
 from galaxy.model.custom_types import total_size
 from galaxy.tool_util.provided_metadata import parse_tool_provided_metadata
-from galaxy.util import (
-    stringify_dictionary_keys,
-    unicodify,
-)
+from galaxy.util import stringify_dictionary_keys
 
 logging.basicConfig()
 log = logging.getLogger(__name__)
@@ -78,10 +74,7 @@ def set_meta_with_tool_provided(dataset_instance, file_dict, set_meta_kwds, data
 
 
 def set_metadata():
-    if len(sys.argv) == 1:
-        set_metadata_portable()
-    else:
-        set_metadata_legacy()
+    set_metadata_portable()
 
 
 def set_metadata_portable():
@@ -172,17 +165,13 @@ def set_metadata_portable():
         job_context = ExpressionContext(dict(stdout=tool_stdout, stderr=tool_stderr))
 
         # Load outputs.
-        import_model_store = store.imported_store_for_metadata('metadata/outputs_new', object_store=object_store)
         export_store = store.DirectoryModelExportStore('metadata/outputs_populated', serialize_dataset_objects=True, for_edit=True, strip_metadata_files=False)
+    import_model_store = store.imported_store_for_metadata('metadata/outputs_new', object_store=object_store)
 
     for output_name, output_dict in outputs.items():
-        if extended_metadata_collection:
-            dataset_instance_id = output_dict["id"]
-            dataset = import_model_store.sa_session.query(galaxy.model.HistoryDatasetAssociation).find(dataset_instance_id)
-            assert dataset is not None
-        else:
-            filename_in = os.path.join("metadata/metadata_in_%s" % output_name)
-            dataset = pickle.load(open(filename_in, 'rb'))  # load DatasetInstance
+        dataset_instance_id = output_dict["id"]
+        dataset = import_model_store.sa_session.query(galaxy.model.HistoryDatasetAssociation).find(dataset_instance_id)
+        assert dataset is not None
 
         filename_kwds = os.path.join("metadata/metadata_kwds_%s" % output_name)
         filename_out = os.path.join("metadata/metadata_out_%s" % output_name)
@@ -311,64 +300,6 @@ def set_metadata_portable():
     write_job_metadata(tool_job_working_directory, job_metadata, set_meta, tool_provided_metadata)
 
 
-def set_metadata_legacy():
-    import galaxy.model
-    galaxy.model.metadata.MetadataTempFile.tmp_dir = tool_job_working_directory = os.path.abspath(os.getcwd())
-
-    # This is ugly, but to transition from existing jobs without this parameter
-    # to ones with, smoothly, it has to be the last optional parameter and we
-    # have to sniff it.
-    try:
-        max_metadata_value_size = int(sys.argv[-1])
-        sys.argv = sys.argv[:-1]
-    except ValueError:
-        max_metadata_value_size = 0
-        # max_metadata_value_size is unspecified and should be 0
-
-    # Set up datatypes registry
-    datatypes_config = sys.argv.pop(1)
-    datatypes_registry = validate_and_load_datatypes_config(datatypes_config)
-
-    job_metadata = sys.argv.pop(1)
-    tool_provided_metadata = load_job_metadata(job_metadata, None)
-
-    def set_meta(new_dataset_instance, file_dict):
-        set_meta_with_tool_provided(new_dataset_instance, file_dict, set_meta_kwds, datatypes_registry, max_metadata_value_size)
-
-    for filenames in sys.argv[1:]:
-        fields = filenames.split(',')
-        filename_in = fields.pop(0)
-        filename_kwds = fields.pop(0)
-        filename_out = fields.pop(0)
-        filename_results_code = fields.pop(0)
-        dataset_filename_override = fields.pop(0)
-        override_metadata = fields.pop(0)
-        set_meta_kwds = stringify_dictionary_keys(json.load(open(filename_kwds)))  # load kwds; need to ensure our keywords are not unicode
-        try:
-            dataset = pickle.load(open(filename_in, 'rb'))  # load DatasetInstance
-            dataset.dataset.external_filename = dataset_filename_override
-            store_by = "id"
-            extra_files_dir_name = "dataset_%s_files" % getattr(dataset.dataset, store_by)
-            files_path = os.path.abspath(os.path.join(tool_job_working_directory, "working", extra_files_dir_name))
-            dataset.dataset.external_extra_files_path = files_path
-            file_dict = tool_provided_metadata.get_dataset_meta(None, dataset.dataset.id, dataset.dataset.uuid)
-            if 'ext' in file_dict:
-                dataset.extension = file_dict['ext']
-            # Metadata FileParameter types may not be writable on a cluster node, and are therefore temporarily substituted with MetadataTempFiles
-            override_metadata = json.load(open(override_metadata))
-            for metadata_name, metadata_file_override in override_metadata:
-                if galaxy.datatypes.metadata.MetadataTempFile.is_JSONified_value(metadata_file_override):
-                    metadata_file_override = galaxy.datatypes.metadata.MetadataTempFile.from_JSON(metadata_file_override)
-                setattr(dataset.metadata, metadata_name, metadata_file_override)
-            set_meta(dataset, file_dict)
-            dataset.metadata.to_JSON_dict(filename_out)  # write out results of set_meta
-            json.dump((True, 'Metadata has been set successfully'), open(filename_results_code, 'wt+'))  # setting metadata has succeeded
-        except Exception as e:
-            json.dump((False, unicodify(e)), open(filename_results_code, 'wt+'))  # setting metadata has failed somehow
-
-    write_job_metadata(tool_job_working_directory, job_metadata, set_meta, tool_provided_metadata)
-
-
 def validate_and_load_datatypes_config(datatypes_config):
     galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir))
 
diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml
index 5471ac209ae..ed024e7d944 100644
--- a/lib/galaxy/webapps/galaxy/config_schema.yml
+++ b/lib/galaxy/webapps/galaxy/config_schema.yml
@@ -2818,7 +2818,7 @@ mapping:
         required: false
         default: directory
         desc: |
-          Determines how metadata will be set. Valid values are `directory`, `extended` and `legacy`.
+          Determines how metadata will be set. Valid values are `directory` and `extended`.
           In extended mode jobs will decide if a tool run failed, the object stores
           configuration is serialized and made available to the job and is used for
           writing output datasets to the object store as part of the job and dynamic
diff --git a/test/unit/tools/test_metadata.py b/test/unit/tools/test_metadata.py
index bbb2dd7db1b..8f802a11071 100644
--- a/test/unit/tools/test_metadata.py
+++ b/test/unit/tools/test_metadata.py
@@ -33,10 +33,6 @@ class MetadataTestCase(unittest.TestCase, tools_support.UsesApp, tools_support.U
         super().tearDown()
         self.metadata_compute_strategy = None
 
-    def test_simple_output_legacy(self):
-        self.app.config.metadata_strategy = "legacy"
-        self._test_simple_output()
-
     def test_simple_output_directory(self):
         self.app.config.metadata_strategy = "directory"
         self._test_simple_output()
@@ -66,10 +62,6 @@ class MetadataTestCase(unittest.TestCase, tools_support.UsesApp, tools_support.U
         assert output_dataset.metadata.data_lines == 2
         assert output_dataset.metadata.sequences == 1
 
-    def test_primary_dataset_output_extension_legacy(self):
-        self.app.config.metadata_strategy = "legacy"
-        self._test_primary_dataset_output_extension()
-
     def test_primary_dataset_output_extension_directory(self):
         self.app.config.metadata_strategy = "directory"
         self._test_primary_dataset_output_extension()
@@ -99,10 +91,6 @@ class MetadataTestCase(unittest.TestCase, tools_support.UsesApp, tools_support.U
         assert output_dataset.metadata.data_lines == 2
         assert output_dataset.metadata.sequences == 1
 
-    def test_primary_dataset_output_metadata_override_legacy(self):
-        self.app.config.metadata_strategy = "legacy"
-        self._test_primary_dataset_output_metadata_override()
-
     def test_primary_dataset_output_metadata_override_directory(self):
         self.app.config.metadata_strategy = "directory"
         self._test_primary_dataset_output_metadata_override()

From f7cc1fd6b2bf81b4369018dc15e58ebe27d4a1d1 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Thu, 4 Feb 2021 12:55:25 +0100
Subject: [PATCH 13/32] Deal with old jobs

---
 lib/galaxy/metadata/set_metadata.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py
index e47adfaafc0..64d8df1c678 100644
--- a/lib/galaxy/metadata/set_metadata.py
+++ b/lib/galaxy/metadata/set_metadata.py
@@ -171,6 +171,11 @@ def set_metadata_portable():
     for output_name, output_dict in outputs.items():
         dataset_instance_id = output_dict["id"]
         dataset = import_model_store.sa_session.query(galaxy.model.HistoryDatasetAssociation).find(dataset_instance_id)
+        if dataset is None:
+            # legacy check for jobs that started before 21.01, remove on 21.05
+            import pickle
+            filename_in = os.path.join("metadata/metadata_in_%s" % output_name)
+            dataset = pickle.load(open(filename_in, 'rb'))  # load DatasetInstance
         assert dataset is not None
 
         filename_kwds = os.path.join("metadata/metadata_kwds_%s" % output_name)

From 47da42e6c7e58dd87be0ae4691c538bd1e0cacba Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Thu, 4 Feb 2021 13:43:32 +0100
Subject: [PATCH 14/32] Sanitize imports in set_metadata.py

Importing when needed was done to speed up metadata setting, however all
methods for setting metadata need to import galaxy.model, which takes up
the bulk of import time (apart from setting up mappers, so that one I
left where it was). I don't think scattered imports are worth the small
benefit that can be had by doing imports inline.
---
 lib/galaxy/metadata/set_metadata.py | 78 +++++++++++++++++------------
 1 file changed, 46 insertions(+), 32 deletions(-)

diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py
index 64d8df1c678..8211d3f345a 100644
--- a/lib/galaxy/metadata/set_metadata.py
+++ b/lib/galaxy/metadata/set_metadata.py
@@ -3,7 +3,7 @@ Execute an external process to set_meta() on a provided list of pickled datasets
 
 This was formerly scripts/set_metadata.py and expects these arguments:
 
-    %prog datatypes_conf.xml job_metadata_file metadata_in,metadata_kwds,metadata_out,metadata_results_code,output_filename_override,metadata_override... max_metadata_value_size
+    %prog datatypes_conf.xml job_metadata_file metadata_kwds,metadata_out,metadata_results_code,output_filename_override,metadata_override... max_metadata_value_size
 
 Galaxy should be importable on sys.path and output_filename_override should be
 set to the path of the dataset on which metadata is being set
@@ -16,20 +16,48 @@ import os
 import sys
 import traceback
 
-from sqlalchemy.orm import clear_mappers
+from pulsar.client.staging import COMMAND_VERSION_FILENAME
 
-import galaxy.model.mapping  # need to load this before we unpickle, in order to setup properties assigned by the mappers
-from galaxy.model import store
+import galaxy.datatypes.registry
+import galaxy.model.mapping
+from galaxy.datatypes import sniff
+from galaxy.datatypes.data import validate
+from galaxy.job_execution.output_collect import (
+    collect_dynamic_outputs,
+    collect_extra_files,
+    collect_primary_datasets,
+    default_exit_code_file,
+    read_exit_code_from,
+    SessionlessJobContext,
+)
+from galaxy.jobs import TOOL_PROVIDED_JOB_METADATA_KEYS
+from galaxy.model import (
+    Dataset,
+    HistoryDatasetAssociation,
+    HistoryDatasetCollectionAssociation,
+    Job,
+    store,
+)
 from galaxy.model.custom_types import total_size
+from galaxy.model.metadata import MetadataTempFile
+from galaxy.objectstore import build_object_store_from_config
+from galaxy.tool_util.output_checker import (
+    check_output,
+    DETECTED_JOB_STATE,
+)
+from galaxy.tool_util.parser.stdio import (
+    ToolStdioExitCode,
+    ToolStdioRegex,
+)
 from galaxy.tool_util.provided_metadata import parse_tool_provided_metadata
 from galaxy.util import stringify_dictionary_keys
+from galaxy.util.expressions import ExpressionContext
 
 logging.basicConfig()
 log = logging.getLogger(__name__)
 
 
 def set_validated_state(dataset_instance):
-    from galaxy.datatypes.data import validate
     datatype_validation = validate(dataset_instance)
 
     dataset_instance.validated_state = datatype_validation.state
@@ -49,7 +77,6 @@ def set_meta_with_tool_provided(dataset_instance, file_dict, set_meta_kwds, data
     extension = dataset_instance.extension
     if extension == "_sniff_":
         try:
-            from galaxy.datatypes import sniff
             extension = sniff.handle_uploaded_dataset_file(dataset_instance.dataset.external_filename, datatypes_registry)
             # We need to both set the extension so it is available to set_meta
             # and record it in the metadata so it can be reloaded on the server
@@ -78,10 +105,9 @@ def set_metadata():
 
 
 def set_metadata_portable():
-    import galaxy.model
     tool_job_working_directory = os.path.abspath(os.getcwd())
     metadata_tmp_files_dir = os.path.join(tool_job_working_directory, "metadata")
-    galaxy.model.metadata.MetadataTempFile.tmp_dir = metadata_tmp_files_dir
+    MetadataTempFile.tmp_dir = metadata_tmp_files_dir
 
     metadata_params_path = os.path.join("metadata", "params.json")
     try:
@@ -110,7 +136,6 @@ def set_metadata_portable():
 
     export_store = None
     if extended_metadata_collection:
-        from galaxy.tool_util.parser.stdio import ToolStdioRegex, ToolStdioExitCode
         tool_dict = metadata_params["tool"]
         stdio_exit_code_dicts, stdio_regex_dicts = tool_dict["stdio_exit_codes"], tool_dict["stdio_regexes"]
         stdio_exit_codes = list(map(ToolStdioExitCode, stdio_exit_code_dicts))
@@ -118,10 +143,9 @@ def set_metadata_portable():
 
         with open(object_store_conf_path) as f:
             config_dict = json.load(f)
-        from galaxy.objectstore import build_object_store_from_config
         assert config_dict is not None
         object_store = build_object_store_from_config(None, config_dict=config_dict)
-        galaxy.model.Dataset.object_store = object_store
+        Dataset.object_store = object_store
 
         outputs_directory = os.path.join(tool_job_working_directory, "outputs")
         if not os.path.exists(outputs_directory):
@@ -144,24 +168,19 @@ def set_metadata_portable():
 
         job_id_tag = metadata_params["job_id_tag"]
 
-        # TODO: this clearly needs to be refactored, nothing in runners should be imported here..
-        from galaxy.job_execution.output_collect import default_exit_code_file, read_exit_code_from
         exit_code_file = default_exit_code_file(".", job_id_tag)
         tool_exit_code = read_exit_code_from(exit_code_file, job_id_tag)
 
-        from galaxy.tool_util.output_checker import check_output, DETECTED_JOB_STATE
         check_output_detected_state, tool_stdout, tool_stderr, job_messages = check_output(stdio_regexes, stdio_exit_codes, tool_stdout, tool_stderr, tool_exit_code, job_id_tag)
         if check_output_detected_state == DETECTED_JOB_STATE.OK and not tool_provided_metadata.has_failed_outputs():
-            final_job_state = galaxy.model.Job.states.OK
+            final_job_state = Job.states.OK
         else:
-            final_job_state = galaxy.model.Job.states.ERROR
+            final_job_state = Job.states.ERROR
 
-        from pulsar.client.staging import COMMAND_VERSION_FILENAME
         version_string = ""
         if os.path.exists(COMMAND_VERSION_FILENAME):
             version_string = open(COMMAND_VERSION_FILENAME).read()
 
-        from galaxy.util.expressions import ExpressionContext
         job_context = ExpressionContext(dict(stdout=tool_stdout, stderr=tool_stderr))
 
         # Load outputs.
@@ -170,11 +189,11 @@ def set_metadata_portable():
 
     for output_name, output_dict in outputs.items():
         dataset_instance_id = output_dict["id"]
-        dataset = import_model_store.sa_session.query(galaxy.model.HistoryDatasetAssociation).find(dataset_instance_id)
+        dataset = import_model_store.sa_session.query(HistoryDatasetAssociation).find(dataset_instance_id)
         if dataset is None:
             # legacy check for jobs that started before 21.01, remove on 21.05
-            import pickle
             filename_in = os.path.join("metadata/metadata_in_%s" % output_name)
+            import pickle
             dataset = pickle.load(open(filename_in, 'rb'))  # load DatasetInstance
         assert dataset is not None
 
@@ -200,8 +219,8 @@ def set_metadata_portable():
             # Metadata FileParameter types may not be writable on a cluster node, and are therefore temporarily substituted with MetadataTempFiles
             override_metadata = json.load(open(override_metadata))
             for metadata_name, metadata_file_override in override_metadata:
-                if galaxy.datatypes.metadata.MetadataTempFile.is_JSONified_value(metadata_file_override):
-                    metadata_file_override = galaxy.datatypes.metadata.MetadataTempFile.from_JSON(metadata_file_override)
+                if MetadataTempFile.is_JSONified_value(metadata_file_override):
+                    metadata_file_override = MetadataTempFile.from_JSON(metadata_file_override)
                 setattr(dataset.metadata, metadata_name, metadata_file_override)
             if output_dict.get("validate", False):
                 set_validated_state(dataset)
@@ -234,9 +253,8 @@ def set_metadata_portable():
                     # This has to be a job with outputs_to_working_directory set.
                     # We update the object store with the created output file.
                     object_store.update_from_file(dataset.dataset, file_name=dataset_filename_override, create=True)
-                from galaxy.job_execution.output_collect import collect_extra_files
                 collect_extra_files(object_store, dataset, ".")
-                if galaxy.model.Job.states.ERROR == final_job_state:
+                if Job.states.ERROR == final_job_state:
                     dataset.blurb = "error"
                     dataset.mark_unhidden()
                 else:
@@ -256,7 +274,6 @@ def set_metadata_portable():
                         # ... and others don't
                         dataset.set_peek()
 
-                from galaxy.jobs import TOOL_PROVIDED_JOB_METADATA_KEYS
                 for context_key in TOOL_PROVIDED_JOB_METADATA_KEYS:
                     if context_key in context:
                         context_value = context[context_key]
@@ -273,7 +290,6 @@ def set_metadata_portable():
 
     if extended_metadata_collection:
         # discover extra outputs...
-        from galaxy.job_execution.output_collect import collect_dynamic_outputs, collect_primary_datasets, SessionlessJobContext
 
         job_context = SessionlessJobContext(
             metadata_params,
@@ -287,10 +303,10 @@ def set_metadata_portable():
 
         output_collections = {}
         for name, output_collection in metadata_params["output_collections"].items():
-            output_collections[name] = import_model_store.sa_session.query(galaxy.model.HistoryDatasetCollectionAssociation).find(output_collection["id"])
+            output_collections[name] = import_model_store.sa_session.query(HistoryDatasetCollectionAssociation).find(output_collection["id"])
         outputs = {}
         for name, output in metadata_params["outputs"].items():
-            outputs[name] = import_model_store.sa_session.query(galaxy.model.HistoryDatasetAssociation).find(output["id"])
+            outputs[name] = import_model_store.sa_session.query(HistoryDatasetAssociation).find(output["id"])
 
         input_ext = json.loads(metadata_params["job_params"].get("__input_ext", '"data"'))
         collect_primary_datasets(
@@ -315,7 +331,6 @@ def validate_and_load_datatypes_config(datatypes_config):
     if not os.path.exists(datatypes_config):
         print("Metadata setting failed because registry.xml [%s] could not be found. You may retry setting metadata." % datatypes_config)
         sys.exit(1)
-    import galaxy.datatypes.registry
     datatypes_registry = galaxy.datatypes.registry.Registry()
     datatypes_registry.load_datatypes(root_dir=galaxy_root, config=datatypes_config, use_build_sites=False, use_converters=False, use_display_applications=False)
     galaxy.model.set_datatypes_registry(datatypes_registry)
@@ -330,14 +345,13 @@ def write_job_metadata(tool_job_working_directory, job_metadata, set_meta, tool_
     for i, file_dict in enumerate(tool_provided_metadata.get_new_datasets_for_metadata_collection(), start=1):
         filename = file_dict["filename"]
         new_dataset_filename = os.path.join(tool_job_working_directory, "working", filename)
-        new_dataset = galaxy.model.Dataset(id=-i, external_filename=new_dataset_filename)
+        new_dataset = Dataset(id=-i, external_filename=new_dataset_filename)
         extra_files = file_dict.get('extra_files', None)
         if extra_files is not None:
             new_dataset._extra_files_path = os.path.join(tool_job_working_directory, "working", extra_files)
         new_dataset.state = new_dataset.states.OK
-        new_dataset_instance = galaxy.model.HistoryDatasetAssociation(id=-i, dataset=new_dataset, extension=file_dict.get('ext', 'data'))
+        new_dataset_instance = HistoryDatasetAssociation(id=-i, dataset=new_dataset, extension=file_dict.get('ext', 'data'))
         set_meta(new_dataset_instance, file_dict)
         file_dict['metadata'] = json.loads(new_dataset_instance.metadata.to_JSON_dict())  # storing metadata in external form, need to turn back into dict, then later jsonify
 
     tool_provided_metadata.rewrite()
-    clear_mappers()

From fb2008943e31cbcdec24063d977e4bac0fd50df5 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Thu, 4 Feb 2021 17:44:41 +0100
Subject: [PATCH 15/32] Fix loading LDDAs in model store

---
 lib/galaxy/metadata/__init__.py     | 1 +
 lib/galaxy/metadata/set_metadata.py | 6 ++++--
 lib/galaxy/model/store/__init__.py  | 9 +++++++--
 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/lib/galaxy/metadata/__init__.py b/lib/galaxy/metadata/__init__.py
index f072015bd55..98f2e29dc4b 100644
--- a/lib/galaxy/metadata/__init__.py
+++ b/lib/galaxy/metadata/__init__.py
@@ -142,6 +142,7 @@ class PortableDirectoryMetadataGenerator(MetadataCollectionStrategy):
                 "validate": validate_outputs,
                 "object_store_store_by": dataset.dataset.store_by,
                 'id': dataset.id,
+                'model_class': 'LibraryDatasetDatasetAssociation' if isinstance(dataset, galaxy.model.LibraryDatasetDatasetAssociation) else 'HistoryDatasetAssociation'
             }
 
         metadata_params_path = os.path.join(metadata_dir, "params.json")
diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py
index 8211d3f345a..c5cb427f6f1 100644
--- a/lib/galaxy/metadata/set_metadata.py
+++ b/lib/galaxy/metadata/set_metadata.py
@@ -189,7 +189,8 @@ def set_metadata_portable():
 
     for output_name, output_dict in outputs.items():
         dataset_instance_id = output_dict["id"]
-        dataset = import_model_store.sa_session.query(HistoryDatasetAssociation).find(dataset_instance_id)
+        klass = getattr(galaxy.model, output_dict.get('model_class', 'HistoryDatasetAssociation'))
+        dataset = import_model_store.sa_session.query(klass).find(dataset_instance_id)
         if dataset is None:
             # legacy check for jobs that started before 21.01, remove on 21.05
             filename_in = os.path.join("metadata/metadata_in_%s" % output_name)
@@ -306,7 +307,8 @@ def set_metadata_portable():
             output_collections[name] = import_model_store.sa_session.query(HistoryDatasetCollectionAssociation).find(output_collection["id"])
         outputs = {}
         for name, output in metadata_params["outputs"].items():
-            outputs[name] = import_model_store.sa_session.query(HistoryDatasetAssociation).find(output["id"])
+            klass = getattr(galaxy.model, output.get('model_class', 'HistoryDatasetAssociation'))
+            outputs[name] = import_model_store.sa_session.query(klass).find(output["id"])
 
         input_ext = json.loads(metadata_params["job_params"].get("__input_ext", '"data"'))
         collect_primary_datasets(
diff --git a/lib/galaxy/model/store/__init__.py b/lib/galaxy/model/store/__init__.py
index 525db57c0d4..41ec4003ae7 100644
--- a/lib/galaxy/model/store/__init__.py
+++ b/lib/galaxy/model/store/__init__.py
@@ -370,7 +370,11 @@ class ModelImportStore(metaclass=abc.ABCMeta):
                         assert 'id' in dataset_attrs
                         object_import_tracker.hdas_by_id[dataset_attrs['id']] = dataset_instance
                 else:
-                    object_import_tracker.lddas_by_key[dataset_attrs[object_key]] = dataset_instance
+                    if object_key in dataset_attrs:
+                        object_import_tracker.lddas_by_key[dataset_attrs[object_key]] = dataset_instance
+                    else:
+                        assert 'id' in dataset_attrs
+                        object_import_tracker.lddas_by_key[dataset_attrs['id']] = dataset_instance
 
     def _import_libraries(self, object_import_tracker):
         object_key = self.object_key
@@ -1251,7 +1255,8 @@ class DirectoryModelExportStore(ModelExportStore):
         def record_associated_jobs(obj):
             # Get the job object.
             job = None
-            for assoc in obj.creating_job_associations:
+            for assoc in getattr(obj, 'creating_job_associations', []):
+                # For mapped over jobs obj could be DatasetCollection, which has no creating_job_association
                 job = assoc.job
                 break
             if not job:

From d3992c37c2aadf3b7ae1eb6f7770514d9b41e80c Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Thu, 4 Feb 2021 17:48:31 +0100
Subject: [PATCH 16/32] Pulsar fix

---
 lib/galaxy/metadata/set_metadata.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py
index c5cb427f6f1..7e90b64dfdc 100644
--- a/lib/galaxy/metadata/set_metadata.py
+++ b/lib/galaxy/metadata/set_metadata.py
@@ -16,7 +16,11 @@ import os
 import sys
 import traceback
 
-from pulsar.client.staging import COMMAND_VERSION_FILENAME
+try:
+    from pulsar.client.staging import COMMAND_VERSION_FILENAME
+except ImportError:
+    # Package unit tests
+    COMMAND_VERSION_FILENAME = 'COMMAND_VERSION'
 
 import galaxy.datatypes.registry
 import galaxy.model.mapping

From 2c98656be4721b8ab261a07771bbbfa32e90ebb9 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Thu, 4 Feb 2021 20:08:04 +0100
Subject: [PATCH 17/32] Fix model store map over collection output

---
 lib/galaxy/job_execution/setup.py   |  3 ++
 lib/galaxy/jobs/__init__.py         | 11 +++-----
 lib/galaxy/metadata/set_metadata.py |  2 +-
 lib/galaxy/model/store/__init__.py  | 43 +++++++++++++++--------------
 4 files changed, 31 insertions(+), 28 deletions(-)

diff --git a/lib/galaxy/job_execution/setup.py b/lib/galaxy/job_execution/setup.py
index 48af3aa1c9f..0e3471215bc 100644
--- a/lib/galaxy/job_execution/setup.py
+++ b/lib/galaxy/job_execution/setup.py
@@ -3,6 +3,9 @@ import os
 
 from galaxy.util import safe_makedirs
 
+TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json'
+TOOL_PROVIDED_JOB_METADATA_KEYS = ['name', 'info', 'dbkey', 'created_from_basename']
+
 
 def ensure_configs_directory(work_dir):
     configs_dir = os.path.join(work_dir, "configs")
diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py
index 6a50aa34d81..6e41dd49074 100644
--- a/lib/galaxy/jobs/__init__.py
+++ b/lib/galaxy/jobs/__init__.py
@@ -40,9 +40,12 @@ from galaxy.job_execution.datasets import (
     TaskPathRewriter
 )
 from galaxy.job_execution.output_collect import collect_extra_files
-from galaxy.job_execution.setup import (
+from galaxy.job_execution.setup import (  # noqa: F401
     create_working_directory_for_job,
     ensure_configs_directory,
+    # This is read by certain misbehaving tool wrappers that import Galaxy internals
+    TOOL_PROVIDED_JOB_METADATA_FILE,
+    TOOL_PROVIDED_JOB_METADATA_KEYS,
 )
 from galaxy.jobs.actions.post import ActionBox
 from galaxy.jobs.mapper import (
@@ -72,12 +75,6 @@ from galaxy.web_stack.handlers import ConfiguresHandlers
 
 log = logging.getLogger(__name__)
 
-# Legacy definition - this is read by certain misbehaving tool wrappers
-# that import Galaxy internals - but it shouldn't be used in Galaxy's code
-# itself.
-TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json'
-TOOL_PROVIDED_JOB_METADATA_KEYS = ['name', 'info', 'dbkey', 'created_from_basename']
-
 # Override with config.default_job_shell.
 DEFAULT_JOB_SHELL = '/bin/bash'
 DEFAULT_LOCAL_WORKERS = 4
diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py
index 7e90b64dfdc..4f95382d8a6 100644
--- a/lib/galaxy/metadata/set_metadata.py
+++ b/lib/galaxy/metadata/set_metadata.py
@@ -34,7 +34,7 @@ from galaxy.job_execution.output_collect import (
     read_exit_code_from,
     SessionlessJobContext,
 )
-from galaxy.jobs import TOOL_PROVIDED_JOB_METADATA_KEYS
+from galaxy.job_execution.setup import TOOL_PROVIDED_JOB_METADATA_KEYS
 from galaxy.model import (
     Dataset,
     HistoryDatasetAssociation,
diff --git a/lib/galaxy/model/store/__init__.py b/lib/galaxy/model/store/__init__.py
index 41ec4003ae7..bc7fb924906 100644
--- a/lib/galaxy/model/store/__init__.py
+++ b/lib/galaxy/model/store/__init__.py
@@ -490,29 +490,32 @@ class ModelImportStore(metaclass=abc.ABCMeta):
             return dc
 
         for collection_attrs in collections_attrs:
-            dc = import_collection(collection_attrs["collection"])
-            if 'id' in collection_attrs and self.import_options.allow_edit and not self.sessionless:
-                hdca = self.sa_session.query(model.HistoryDatasetCollectionAssociation).get(collection_attrs["id"])
-                # TODO: edit attributes...
-            else:
-                hdca = model.HistoryDatasetCollectionAssociation(collection=dc,
-                                                                 visible=True,
-                                                                 name=collection_attrs['display_name'],
-                                                                 implicit_output_name=collection_attrs.get("implicit_output_name"))
-                self._attach_raw_id_if_editing(hdca, collection_attrs)
-
-                hdca.history = history
-                if new_history and self.trust_hid(collection_attrs):
-                    hdca.hid = collection_attrs['hid']
+            if 'collection' in collection_attrs:
+                dc = import_collection(collection_attrs["collection"])
+                if 'id' in collection_attrs and self.import_options.allow_edit and not self.sessionless:
+                    hdca = self.sa_session.query(model.HistoryDatasetCollectionAssociation).get(collection_attrs["id"])
+                    # TODO: edit attributes...
                 else:
-                    object_import_tracker.requires_hid.append(hdca)
+                    hdca = model.HistoryDatasetCollectionAssociation(collection=dc,
+                                                                     visible=True,
+                                                                     name=collection_attrs['display_name'],
+                                                                     implicit_output_name=collection_attrs.get("implicit_output_name"))
+                    self._attach_raw_id_if_editing(hdca, collection_attrs)
 
-            self._session_add(hdca)
-            if object_key in collection_attrs:
-                object_import_tracker.hdcas_by_key[collection_attrs[object_key]] = hdca
+                    hdca.history = history
+                    if new_history and self.trust_hid(collection_attrs):
+                        hdca.hid = collection_attrs['hid']
+                    else:
+                        object_import_tracker.requires_hid.append(hdca)
+
+                self._session_add(hdca)
+                if object_key in collection_attrs:
+                    object_import_tracker.hdcas_by_key[collection_attrs[object_key]] = hdca
+                else:
+                    assert 'id' in collection_attrs
+                    object_import_tracker.hdcas_by_id[collection_attrs['id']] = hdca
             else:
-                assert 'id' in collection_attrs
-                object_import_tracker.hdcas_by_id[collection_attrs['id']] = hdca
+                import_collection(collection_attrs)
 
     def _attach_raw_id_if_editing(self, obj, attrs):
         if self.sessionless and 'id' in attrs and self.import_options.allow_edit:

From 029c017c690a76a4535416d899f56412fadf7126 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 5 Feb 2021 13:10:19 +0100
Subject: [PATCH 18/32] Drop strip_control_characters_nested workaround

Pretty sure https://github.com/galaxyproject/galaxy/pull/10390/ fixed
the null issue to start with.
---
 lib/galaxy/util/__init__.py    | 13 -------------
 lib/galaxy/util/rules_dsl.py   |  4 +---
 lib/galaxy/workflow/modules.py | 10 ++++------
 test/unit/util/test_utils.py   | 11 -----------
 4 files changed, 5 insertions(+), 33 deletions(-)

diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py
index 4637f00671d..1c2bbf19d9e 100644
--- a/lib/galaxy/util/__init__.py
+++ b/lib/galaxy/util/__init__.py
@@ -1086,19 +1086,6 @@ def strip_control_characters(s):
     return "".join(c for c in unicodify(s) if unicodedata.category(c) != "Cc")
 
 
-def strip_control_characters_nested(item):
-    """Recursively strips control characters from lists, dicts, tuples."""
-
-    def visit(path, key, value):
-        if isinstance(key, str):
-            key = strip_control_characters(key)
-        if isinstance(value, str):
-            value = strip_control_characters(value)
-        return key, value
-
-    return remap(item, visit)
-
-
 def object_to_string(obj):
     return binascii.hexlify(obj)
 
diff --git a/lib/galaxy/util/rules_dsl.py b/lib/galaxy/util/rules_dsl.py
index f13b3c43dc9..d946d0d7333 100644
--- a/lib/galaxy/util/rules_dsl.py
+++ b/lib/galaxy/util/rules_dsl.py
@@ -6,8 +6,6 @@ from typing import List, Type
 import yaml
 from pkg_resources import resource_stream
 
-from galaxy.util import strip_control_characters_nested
-
 
 def get_rules_specification():
     return yaml.safe_load(resource_stream(__name__, 'rules_dsl_spec.yml'))
@@ -498,7 +496,7 @@ def flat_map(f, items):
 class RuleSet:
 
     def __init__(self, rule_set_as_dict):
-        self.raw_rules = strip_control_characters_nested(rule_set_as_dict["rules"])
+        self.raw_rules = rule_set_as_dict["rules"]
         self.raw_mapping = rule_set_as_dict.get("mapping", [])
 
     @property
diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py
index ef259d9cd73..aa7faa21d91 100644
--- a/lib/galaxy/workflow/modules.py
+++ b/lib/galaxy/workflow/modules.py
@@ -1445,12 +1445,10 @@ class ToolModule(WorkflowModule):
                     if not collection_type and tool_output.structure.collection_type_from_rules:
                         rule_param = tool_output.structure.collection_type_from_rules
                         if rule_param in self.state.inputs:
-                            rule_json_str = self.state.inputs[rule_param]
-                            if rule_json_str:  # initialized to None...
-                                rules = rule_json_str
-                                if rules:
-                                    rule_set = RuleSet(rules)
-                                    collection_type = rule_set.collection_type
+                            rules = self.state.inputs[rule_param]
+                            if rules:
+                                rule_set = RuleSet(rules)
+                                collection_type = rule_set.collection_type
                     extra_kwds["collection_type"] = collection_type
                     extra_kwds["collection_type_source"] = tool_output.structure.collection_type_source
                     formats = ['input']  # TODO: fix
diff --git a/test/unit/util/test_utils.py b/test/unit/util/test_utils.py
index 08ffd0fc04a..8846b26da22 100644
--- a/test/unit/util/test_utils.py
+++ b/test/unit/util/test_utils.py
@@ -21,17 +21,6 @@ def test_strip_control_characters():
     assert util.strip_control_characters(s) == 'bla'
 
 
-def test_strip_control_characters_nested():
-    s = '\x00bla'
-    stripped_s = 'bla'
-    l = [s]
-    t = (s, 'blub')
-    d = {42: s}
-    assert util.strip_control_characters_nested(l)[0] == stripped_s
-    assert util.strip_control_characters_nested(t)[0] == stripped_s
-    assert util.strip_control_characters_nested(d)[42] == stripped_s
-
-
 def test_parse_xml_string():
     section = util.parse_xml_string(SECTION_XML)
     _verify_section(section)

From c05ebd88ba0769c485dfd401c73582e86bba8996 Mon Sep 17 00:00:00 2001
From: Dannon Baker <dannon.baker@gmail.com>
Date: Fri, 5 Feb 2021 09:20:55 -0500
Subject: [PATCH 19/32] Change initializations group in console to not collapse
 by default (so errors are visible)

---
 client/src/onload/standardInit.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/onload/standardInit.js b/client/src/onload/standardInit.js
index 9339af223d3..5bddd047559 100644
--- a/client/src/onload/standardInit.js
+++ b/client/src/onload/standardInit.js
@@ -45,7 +45,7 @@ export function standardInit(label = "Galaxy", appFactory = defaultAppFactory) {
     // will not remake a the existing Galaxy or config objects, it'll just run
     // the new batch of freshly registered init functions
     combineLatest(config$, galaxy$, initializations$).subscribe(([config, galaxy, inits]) => {
-        console.groupCollapsed(`runInitializations`, label, serverPath());
+        console.group(`runInitializations`, label, serverPath());
         inits.forEach((fn) => fn(galaxy, config));
         clearInitQueue();
         console.groupEnd();

From aae44a6657d90abcb204af79c657c5cdebff9cd9 Mon Sep 17 00:00:00 2001
From: Dannon Baker <dannon.baker@gmail.com>
Date: Fri, 5 Feb 2021 09:21:56 -0500
Subject: [PATCH 20/32] Update combineLatest arguments (was deprecated syntax)

---
 client/src/onload/standardInit.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/onload/standardInit.js b/client/src/onload/standardInit.js
index 5bddd047559..91467d007da 100644
--- a/client/src/onload/standardInit.js
+++ b/client/src/onload/standardInit.js
@@ -44,7 +44,7 @@ export function standardInit(label = "Galaxy", appFactory = defaultAppFactory) {
     // functions even if they are registered super-late because combineLatest
     // will not remake a the existing Galaxy or config objects, it'll just run
     // the new batch of freshly registered init functions
-    combineLatest(config$, galaxy$, initializations$).subscribe(([config, galaxy, inits]) => {
+    combineLatest([config$, galaxy$, initializations$]).subscribe(([config, galaxy, inits]) => {
         console.group(`runInitializations`, label, serverPath());
         inits.forEach((fn) => fn(galaxy, config));
         clearInitQueue();

From 356fd5c3689ce16ce6c44666b69710e7b1393ee3 Mon Sep 17 00:00:00 2001
From: Dannon Baker <dannon.baker@gmail.com>
Date: Fri, 5 Feb 2021 10:04:28 -0500
Subject: [PATCH 21/32] Bugfix- on initial share toggle this was never updated,
 causing the 'null' slugs

---
 client/src/components/Sharing.vue | 1 +
 1 file changed, 1 insertion(+)

diff --git a/client/src/components/Sharing.vue b/client/src/components/Sharing.vue
index 68ff43031bd..a2fdc619e3c 100644
--- a/client/src/components/Sharing.vue
+++ b/client/src/components/Sharing.vue
@@ -259,6 +259,7 @@ export default {
                     if (response.data.skipped) {
                         this.errMsg = "Some of the items within this object were not published due to an error.";
                     }
+                    this.item = response.data;
                 })
                 .catch((error) => (this.errMsg = error.response.data.err_msg));
         },

From 7f1a97d1befac80e97c68e2a3c110358b20fe92c Mon Sep 17 00:00:00 2001
From: Dannon Baker <dannon.baker@gmail.com>
Date: Fri, 5 Feb 2021 10:24:38 -0500
Subject: [PATCH 22/32] Require slugInput prop

---
 client/src/components/Common/SlugInput.vue | 1 +
 1 file changed, 1 insertion(+)

diff --git a/client/src/components/Common/SlugInput.vue b/client/src/components/Common/SlugInput.vue
index 6b102344fec..84cb33e0e99 100644
--- a/client/src/components/Common/SlugInput.vue
+++ b/client/src/components/Common/SlugInput.vue
@@ -18,6 +18,7 @@ export default {
     props: {
         slug: {
             type: String,
+            required: true
         },
     },
     data() {

From 137899677fe2a4c6ed45f4b40ac36b4ed83a3708 Mon Sep 17 00:00:00 2001
From: Dannon Baker <dannon.baker@gmail.com>
Date: Fri, 5 Feb 2021 10:26:23 -0500
Subject: [PATCH 23/32] Minor cleanup in sharing component

---
 client/src/components/Sharing.vue | 60 +++++++++++++++----------------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/client/src/components/Sharing.vue b/client/src/components/Sharing.vue
index a2fdc619e3c..73337cd6467 100644
--- a/client/src/components/Sharing.vue
+++ b/client/src/components/Sharing.vue
@@ -12,7 +12,7 @@
         </div>
         <div v-else>
             <b-form-checkbox switch class="make-accessible" v-model="item.importable" @change="onImportable">
-                Make {{ model_class }} accessible.
+                Make {{ model_class }} accessible
             </b-form-checkbox>
             <b-form-checkbox
                 v-if="item.importable"
@@ -22,38 +22,38 @@
                 @change="onPublish"
             >
                 Make {{ model_class }} publicly available in
-                <a :href="published_url" target="_top">Published {{ plural_name }}</a> section.
+                <a :href="published_url" target="_top">Published {{ plural_name }}</a>
             </b-form-checkbox>
             <br />
-            <div>
-                <div v-if="item.importable">
-                    <div>
-                        This {{ model_class }} is currently <strong>{{ itemStatus }}</strong
-                        >.
-                    </div>
-                    <p>Anyone can view and import this {{ model_class }} by visiting the following URL:</p>
-                    <blockquote>
-                        <b-button title="Edit URL" @click="onEdit" v-b-tooltip.hover variant="link" size="sm">
-                            <font-awesome-icon icon="edit" />
-                        </b-button>
-                        <b-button id="tooltip-clipboard" @click="onCopy" @mouseout="onCopyOut" variant="link" size="sm">
-                            <font-awesome-icon icon="link" />
-                        </b-button>
-                        <b-tooltip target="tooltip-clipboard" triggers="hover">
-                            {{ tooltipClipboard }}
-                        </b-tooltip>
-                        <a v-if="showUrl" id="item-url" :href="itemUrl" target="_top" class="ml-2">
-                            {{ itemUrl }}
-                        </a>
-                        <span v-else id="item-url-text">
-                            {{ itemUrlParts[0] }}<SlugInput class="ml-1" :slug="itemUrlParts[1]" @onChange="onChange" />
-                        </span>
-                    </blockquote>
-                </div>
-                <div v-else>
-                    Access to this {{ model_class }} is currently restricted so that only you and the users listed below
-                    can access it. Note that sharing a History will also allow access to all of its datasets.
+            <div v-if="item.importable">
+                <div>
+                    This {{ model_class }} is currently <strong>{{ itemStatus }}</strong
+                    >.
                 </div>
+                <p>Anyone can view and import this {{ model_class }} by visiting the following URL:</p>
+                <blockquote>
+                    <b-button title="Edit URL" @click="onEdit" v-b-tooltip.hover variant="link" size="sm">
+                        <font-awesome-icon icon="edit" />
+                    </b-button>
+                    <b-button id="tooltip-clipboard" @click="onCopy" @mouseout="onCopyOut" variant="link" size="sm">
+                        <font-awesome-icon icon="link" />
+                    </b-button>
+                    <b-tooltip target="tooltip-clipboard" triggers="hover">
+                        {{ tooltipClipboard }}
+                    </b-tooltip>
+                    <a v-if="showUrl" id="item-url" :href="itemUrl" target="_top" class="ml-2">
+                        url:
+                        {{ itemUrl }}
+                    </a>
+                    <span v-else id="item-url-text">
+                        slug:
+                        {{ itemUrlParts[0] }}<SlugInput class="ml-1" :slug="itemUrlParts[1]" @onChange="onChange" />
+                    </span>
+                </blockquote>
+            </div>
+            <div v-else>
+                Access to this {{ model_class }} is currently restricted so that only you and the users listed below can
+                access it. Note that sharing a History will also allow access to all of its datasets.
             </div>
             <br />
             <h4>Share {{ model_class }} with Individual Users</h4>

From 20d5eba54308578c0f77361326d968b11b352e6a Mon Sep 17 00:00:00 2001
From: Dannon Baker <dannon.baker@gmail.com>
Date: Fri, 5 Feb 2021 10:40:42 -0500
Subject: [PATCH 24/32] Make setSharing async, chain requests to avoid order of
 ops errors and mitigate the 'flash' when initially setting due to item
 computed props recalculating simultaneously

---
 client/src/components/Common/SlugInput.vue |  2 +-
 client/src/components/Sharing.vue          | 26 +++++++++++++---------
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/client/src/components/Common/SlugInput.vue b/client/src/components/Common/SlugInput.vue
index 84cb33e0e99..27c098eaa22 100644
--- a/client/src/components/Common/SlugInput.vue
+++ b/client/src/components/Common/SlugInput.vue
@@ -18,7 +18,7 @@ export default {
     props: {
         slug: {
             type: String,
-            required: true
+            required: true,
         },
     },
     data() {
diff --git a/client/src/components/Sharing.vue b/client/src/components/Sharing.vue
index 73337cd6467..69fe2ea29a8 100644
--- a/client/src/components/Sharing.vue
+++ b/client/src/components/Sharing.vue
@@ -127,7 +127,7 @@ export default {
             errMsg: null,
             item: {
                 title: "title",
-                username_and_slug: "username_and_slug",
+                username_and_slug: "username/slug",
                 importable: false,
                 published: false,
                 users_shared_with: [],
@@ -205,16 +205,19 @@ export default {
         },
         onImportable(importable) {
             if (importable) {
-                this.setSharing("make_accessible_via_link");
-                if (this.item.published) {
-                    this.setSharing("publish");
-                } else {
-                    this.setSharing("unpublish");
-                }
+                const alsoPublish = this.item.published;
+                this.setSharing("make_accessible_via_link").then(() => {
+                    if (alsoPublish) {
+                        this.setSharing("publish");
+                    } else {
+                        this.setSharing("unpublish");
+                    }
+                });
             } else {
                 this.item.published = false;
-                this.setSharing("disable_link_access");
-                this.setSharing("unpublish");
+                this.setSharing("disable_link_access").then(() => {
+                    this.setSharing("unpublish");
+                });
             }
         },
         onPublish(published) {
@@ -248,18 +251,19 @@ export default {
                 })
                 .catch((error) => (this.errMsg = error.response.data.err_msg));
         },
-        setSharing(action, user_id) {
+        async setSharing(action, user_id) {
             const data = {
                 action: action,
                 user_id: user_id,
             };
-            axios
+            return axios
                 .post(`${getAppRoot()}api/${this.pluralNameLower}/${this.id}/sharing`, data)
                 .then((response) => {
                     if (response.data.skipped) {
                         this.errMsg = "Some of the items within this object were not published due to an error.";
                     }
                     this.item = response.data;
+                    this.ready = true;
                 })
                 .catch((error) => (this.errMsg = error.response.data.err_msg));
         },

From 31ee91b1999666afd2d689fe838ae9b72c7d2841 Mon Sep 17 00:00:00 2001
From: Dannon Baker <dannon.baker@gmail.com>
Date: Fri, 5 Feb 2021 11:52:20 -0500
Subject: [PATCH 25/32] Enhance sharing API to allow compound actions (existing
 calls will work exactly as before), eliminate redundant calls in sharing
 component

---
 client/src/components/Sharing.vue     | 15 +++------------
 lib/galaxy/webapps/base/controller.py |  6 ++++--
 2 files changed, 7 insertions(+), 14 deletions(-)

diff --git a/client/src/components/Sharing.vue b/client/src/components/Sharing.vue
index 69fe2ea29a8..d86a78f1355 100644
--- a/client/src/components/Sharing.vue
+++ b/client/src/components/Sharing.vue
@@ -205,19 +205,10 @@ export default {
         },
         onImportable(importable) {
             if (importable) {
-                const alsoPublish = this.item.published;
-                this.setSharing("make_accessible_via_link").then(() => {
-                    if (alsoPublish) {
-                        this.setSharing("publish");
-                    } else {
-                        this.setSharing("unpublish");
-                    }
-                });
+                this.setSharing(`make_accessible_via_link-${this.item.published ? "publish" : "unpublish"}`);
             } else {
                 this.item.published = false;
-                this.setSharing("disable_link_access").then(() => {
-                    this.setSharing("unpublish");
-                });
+                this.setSharing("disable_link_access-unpublish");
             }
         },
         onPublish(published) {
@@ -251,7 +242,7 @@ export default {
                 })
                 .catch((error) => (this.errMsg = error.response.data.err_msg));
         },
-        async setSharing(action, user_id) {
+        setSharing(action, user_id) {
             const data = {
                 action: action,
                 user_id: user_id,
diff --git a/lib/galaxy/webapps/base/controller.py b/lib/galaxy/webapps/base/controller.py
index aab9eddcf33..fc828e15b48 100644
--- a/lib/galaxy/webapps/base/controller.py
+++ b/lib/galaxy/webapps/base/controller.py
@@ -1411,8 +1411,10 @@ class SharableMixin:
         skipped = False
         class_name = self.manager.model_class.__name__
         item = self.get_object(trans, id, class_name, check_ownership=True, check_accessible=True, deleted=False)
-        if payload and payload.get("action"):
-            action = payload.get("action")
+        actions = []
+        if payload:
+            actions += payload.get("action").split("-")
+        for action in actions:
             if action == "make_accessible_via_link":
                 self._make_item_accessible(trans.sa_session, item)
                 if hasattr(item, "has_possible_members") and item.has_possible_members:

From b47a80004f6e1cd7b698023e4ed614f61fc67611 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Fri, 5 Feb 2021 13:08:55 -0500
Subject: [PATCH 26/32] Fix concrete objectstore methods for new/discarded
 datasets.

---
 lib/galaxy/objectstore/__init__.py        | 10 ++++++++--
 test/unit/objectstore/test_objectstore.py |  9 +++++++++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py
index d967c0df1a8..c59d7a2a603 100644
--- a/lib/galaxy/objectstore/__init__.py
+++ b/lib/galaxy/objectstore/__init__.py
@@ -183,6 +183,9 @@ class ObjectStore(metaclass=abc.ABCMeta):
 
         To accommodate nested objectstores, obj is passed in so this metadata can
         be returned for the ConcreteObjectStore corresponding to the object.
+
+        If the dataset is in a new or discarded state and an object_store_id has not
+        yet been set, this may return ``None``.
         """
 
     @abc.abstractmethod
@@ -191,6 +194,9 @@ class ObjectStore(metaclass=abc.ABCMeta):
 
         To accommodate nested objectstores, obj is passed in so this metadata can
         be returned for the ConcreteObjectStore corresponding to the object.
+
+        If the dataset is in a new or discarded state and an object_store_id has not
+        yet been set, this may return ``None``.
         """
 
     @abc.abstractmethod
@@ -701,10 +707,10 @@ class NestedObjectStore(BaseObjectStore):
         return self._call_method('_get_object_url', obj, None, False, **kwargs)
 
     def _get_concrete_store_name(self, obj):
-        return self._call_method('_get_concrete_store_name', obj, None, True)
+        return self._call_method('_get_concrete_store_name', obj, None, False)
 
     def _get_concrete_store_description_markdown(self, obj):
-        return self._call_method('_get_concrete_store_description_markdown', obj, None, True)
+        return self._call_method('_get_concrete_store_description_markdown', obj, None, False)
 
     def _get_store_by(self, obj):
         return self._call_method('_get_store_by', obj, None, False)
diff --git a/test/unit/objectstore/test_objectstore.py b/test/unit/objectstore/test_objectstore.py
index 7102dac0abf..6809cf71f45 100644
--- a/test/unit/objectstore/test_objectstore.py
+++ b/test/unit/objectstore/test_objectstore.py
@@ -273,6 +273,15 @@ def test_hierarchical_store():
             _assert_key_has_value(as_dict, "type", "hierarchical")
 
 
+def test_concrete_name_without_objectstore_id():
+    for config_str in [HIERARCHICAL_TEST_CONFIG, HIERARCHICAL_TEST_CONFIG_YAML]:
+        with TestConfig(config_str) as (directory, object_store):
+            files1_desc = object_store.get_concrete_store_description_markdown(MockDataset(3))
+            files1_name = object_store.get_concrete_store_name(MockDataset(3))
+            assert files1_desc is None
+            assert files1_name is None
+
+
 MIXED_STORE_BY_HIERARCHICAL_TEST_CONFIG = """<?xml version="1.0"?>
 <object_store type="hierarchical">
     <backends>

From 822ca308c1033aa2a241ca09684087fcdfda57b6 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 5 Feb 2021 19:38:52 +0100
Subject: [PATCH 27/32] Fix pulsar test case

---
 lib/galaxy/jobs/runners/pulsar.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py
index be288df7e86..9d607a22acd 100644
--- a/lib/galaxy/jobs/runners/pulsar.py
+++ b/lib/galaxy/jobs/runners/pulsar.py
@@ -593,6 +593,8 @@ class PulsarJobRunner(AsynchronousJobRunner):
             files_endpoint=files_endpoint,
             env=env
         )
+        # Turn MutableDict into standard dict for pulsar consumption
+        job_destination_params = dict(job_destination_params.items())
         return self.client_manager.get_client(job_destination_params, **get_client_kwds)
 
     def finish_job(self, job_state):

From 8fdf5e3a2a6f9142cc1657c4074a1f7e96d4009e Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 7 Feb 2021 14:43:16 +0100
Subject: [PATCH 28/32] Fix populated state for empty collections

This matches the non-optimized variant. If there's an empty collection
(because nothing has been discovered for instance)
DatasetCollection.populated would be true and
DatasetCollection.populated_optimized would be false. This is because of
the way we build the query. Noticed this with a workflow that worked
before https://github.com/galaxyproject/galaxy/pull/10917, which
switched the inputs ready check for database colletion tools to use the
populated_optimized method.
---
 lib/galaxy/model/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index c4d981fef72..fec10af6eb0 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -4012,7 +4012,7 @@ class DatasetCollection(Dictifiable, UsesAnnotations, RepresentById):
                 select_stmt = select(list(map(lambda dc: dc.c.populated_state, collection_depth_aliases))).select_from(select_from).where(dc.c.id == self.id).distinct()
                 for populated_states in db_session.execute(select_stmt).fetchall():
                     for populated_state in populated_states:
-                        if populated_state != DatasetCollection.populated_states.OK:
+                        if populated_state and populated_state != DatasetCollection.populated_states.OK:
                             _populated_optimized = False
 
             self._populated_optimized = _populated_optimized

From 5f7ab67696f54683789e15a594837440a59e9f18 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 8 Feb 2021 11:19:41 +0100
Subject: [PATCH 29/32] Deploy postgresql and rabbitmq in kubernetes

Setting up minikube stops the postgres and rabbitmq containers that are
set up by the workflow. These are setup with custom docker networks, so
it isn't trivial to restart them. Instead we just deploy them to
minikube.
---
 .ci/minikube-test-setup/deployment.yaml   | 32 ++++++++
 .ci/minikube-test-setup/start_services.sh | 12 +++
 .github/workflows/integration.yaml        | 91 +++++++++++++----------
 3 files changed, 95 insertions(+), 40 deletions(-)
 create mode 100644 .ci/minikube-test-setup/deployment.yaml
 create mode 100644 .ci/minikube-test-setup/start_services.sh

diff --git a/.ci/minikube-test-setup/deployment.yaml b/.ci/minikube-test-setup/deployment.yaml
new file mode 100644
index 00000000000..d7afe110c83
--- /dev/null
+++ b/.ci/minikube-test-setup/deployment.yaml
@@ -0,0 +1,32 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: testing
+  name: testing
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: test
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: test
+    spec:
+      containers:
+      - image: postgres:12
+        name: postgres
+        ports:
+        - containerPort: 5432
+        env:
+          - name: POSTGRES_DB
+            value: postgres
+          - name: POSTGRES_USER
+            value:  postgres
+          - name: POSTGRES_PASSWORD
+            value: postgres
+      - image: rabbitmq
+        name: rabbitmq
+        ports:
+        - containerPort: 5672
diff --git a/.ci/minikube-test-setup/start_services.sh b/.ci/minikube-test-setup/start_services.sh
new file mode 100644
index 00000000000..1e860dadcb7
--- /dev/null
+++ b/.ci/minikube-test-setup/start_services.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -ex
+
+SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}")
+kubectl apply -f "$SCRIPTDIR/deployment.yaml"
+kubectl expose deployment testing --type=LoadBalancer --name=testing-service
+
+CLUSTER_IP=$(kubectl get service testing-service -o jsonpath='{.spec.clusterIP}')
+GALAXY_TEST_DBURI="postgresql://postgres:postgres@${CLUSTER_IP}:5432/galaxy?client_encoding=utf-8"
+GALAXY_TEST_AMQP_URL="amqp://${CLUSTER_IP}:5672)//"
+export GALAXY_TEST_DBURI
+export GALAXY_TEST_AMQP_URL
diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
index 22d6d3a7e06..a1cad368b46 100644
--- a/.github/workflows/integration.yaml
+++ b/.github/workflows/integration.yaml
@@ -6,14 +6,15 @@ env:
 jobs:
   test:
     name: Test
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        python-version: [3.7]
-        subset: ['upload_datatype', 'extended_metadata', 'kubernetes', 'not (upload_datatype or extended_metadata or kubernetes)']
+        python-version: ['3.7']
+        subset: ['kubernetes']
     services:
       postgres:
-        image: postgres:11
+        image: postgres:13
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
@@ -25,39 +26,49 @@ jobs:
         ports:
           - 5672:5672
     steps:
-    - name: Prune unused docker image, volumes and containers
-      run: docker system prune -a -f
-    - name: Clean dotnet folder for space
-      if: matrix.subset == 'kubernetes'
-      run: rm -Rf /usr/share/dotnet
-    - name: Setup Minikube
-      if: matrix.subset == 'kubernetes'
-      id: minikube
-      uses: CodingNagger/minikube-setup-action@v1.0.3
-      with:
-        minikube-version: "1.9.0-0_amd64"
-    - name: Launch Minikube
-      if: matrix.subset == 'kubernetes'
-      run: eval ${{ steps.minikube.outputs.launcher }}
-    - name: Check pods
-      if: matrix.subset == 'kubernetes'
-      run: |
-        kubectl get pods
-    - uses: actions/checkout@v2
-      with:
-        path: 'galaxy root'
-    - uses: actions/setup-python@v1
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Cache pip dir
-      uses: actions/cache@v1
-      id: pip-cache
-      with:
-        path: ~/.cache/pip
-        key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
-    - name: Install ffmpeg
-      run: sudo apt-get update && sudo apt-get install ffmpeg -y
-      if: matrix.subset == 'upload_datatype'
-    - name: Run tests
-      run: './run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"'
-      working-directory: 'galaxy root'
+      - name: Prune unused docker image, volumes and containers
+        run: docker system prune -a -f
+      - name: Clean dotnet folder for space
+        if: matrix.subset == 'kubernetes'
+        run: rm -Rf /usr/share/dotnet
+      - name: Setup Minikube
+        if: matrix.subset == 'kubernetes'
+        id: minikube
+        uses: CodingNagger/minikube-setup-action@v1.0.3
+        with:
+          minikube-version: "1.9.0-0_amd64"
+      - name: Launch Minikube
+        if: matrix.subset == 'kubernetes'
+        run: eval ${{ steps.minikube.outputs.launcher }}
+      - name: Check pods
+        if: matrix.subset == 'kubernetes'
+        run: |
+          kubectl get pods
+      - uses: actions/checkout@v2
+        with:
+          path: 'galaxy root'
+      - uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache pip dir
+        uses: actions/cache@v1
+        id: pip-cache
+        with:
+          path: ~/.cache/pip
+          key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+      - name: Install ffmpeg
+        run: sudo apt-get update && sudo apt-get install ffmpeg -y
+        if: matrix.subset == 'upload_datatype'
+      - name: Run tests
+        if: matrix.subset != 'kubernetes'
+        run: './run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"'
+        working-directory: 'galaxy root'
+      - name: Run tests
+        if: matrix.subset == 'kubernetes'
+        run: 'source .ci/minikube-test-setup/start_services.sh && ./run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"'
+        working-directory: 'galaxy root'
+      - uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: Integration test results (${{ matrix.python-version }}, ${{ matrix.subset }})
+          path: 'galaxy root/run_integration_tests.html'

From 12e52783de7a52e843cbd0c5cf6dbb22a3ec8417 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 8 Feb 2021 13:08:45 +0100
Subject: [PATCH 30/32] Restore all integration subsets

---
 .github/workflows/integration.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
index a1cad368b46..1eaf6218d8e 100644
--- a/.github/workflows/integration.yaml
+++ b/.github/workflows/integration.yaml
@@ -11,7 +11,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ['3.7']
-        subset: ['kubernetes']
+        subset: ['upload_datatype', 'extended_metadata', 'kubernetes', 'not (upload_datatype or extended_metadata or kubernetes)']
     services:
       postgres:
         image: postgres:13

From fa9c9bd517b38561c0981d58c71f9328e8f3b5a1 Mon Sep 17 00:00:00 2001
From: Marius van den Beek <m.vandenbeek@gmail.com>
Date: Mon, 8 Feb 2021 14:01:28 +0100
Subject: [PATCH 31/32] Use multiline yaml

Co-authored-by: Nicola Soranzo <nicola.soranzo@gmail.com>
---
 .github/workflows/integration.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
index 1eaf6218d8e..5bce55d5b3e 100644
--- a/.github/workflows/integration.yaml
+++ b/.github/workflows/integration.yaml
@@ -65,7 +65,9 @@ jobs:
         working-directory: 'galaxy root'
       - name: Run tests
         if: matrix.subset == 'kubernetes'
-        run: 'source .ci/minikube-test-setup/start_services.sh && ./run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"'
+        run: |
+          . .ci/minikube-test-setup/start_services.sh
+          ./run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"
         working-directory: 'galaxy root'
       - uses: actions/upload-artifact@v2
         if: failure()

From 25d9f03cec15555b064cec19f7e378c2013aca83 Mon Sep 17 00:00:00 2001
From: Dannon Baker <dannon.baker@gmail.com>
Date: Mon, 8 Feb 2021 09:24:29 -0500
Subject: [PATCH 32/32] minor cleanup when testing

---
 config/plugins/webhooks/gtn/script.js | 35 +++++++++++++++------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/config/plugins/webhooks/gtn/script.js b/config/plugins/webhooks/gtn/script.js
index 0d395ab2052..29d9c1fa660 100644
--- a/config/plugins/webhooks/gtn/script.js
+++ b/config/plugins/webhooks/gtn/script.js
@@ -9,14 +9,14 @@ function showOverlay() {
     document.getElementById("gtn-container").style.visibility = "visible";
 }
 
-function getIframeUrl(){
+function getIframeUrl() {
     var loc;
     try {
         loc = document.getElementById("gtn-embed").contentWindow.location.pathname;
     } catch (e) {
         loc = null;
     }
-    return loc
+    return loc;
 }
 
 function getIframeScroll() {
@@ -26,20 +26,19 @@ function getIframeScroll() {
     } catch (e) {
         loc = 0;
     }
-    return loc
+    return loc;
 }
 
-function restoreLocation() {
-}
+function restoreLocation() {}
 
 function persistLocation() {
     // Don't save every scroll event.
     var time = new Date().getTime();
-    if ( time - lastUpdate < 1000 ) {
+    if (time - lastUpdate < 1000) {
         return;
     }
     lastUpdate = time;
-    window.localStorage.setItem('gtn-in-galaxy', `${getIframeScroll()} ${getIframeUrl()}`);
+    window.localStorage.setItem("gtn-in-galaxy", `${getIframeScroll()} ${getIframeUrl()}`);
 }
 
 function addIframe() {
@@ -62,10 +61,14 @@ function addIframe() {
             } else {
                 safe = true;
 
-                var storedLocation = window.localStorage.getItem('gtn-in-galaxy');
-                if(storedLocation !== null && storedLocation.split(' ')[1] !== undefined && storedLocation.split(' ')[1].startsWith('/training-material/')) {
-                    onloadscroll = storedLocation.split(' ')[0];
-                    url = storedLocation.split(' ')[1];
+                var storedLocation = window.localStorage.getItem("gtn-in-galaxy");
+                if (
+                    storedLocation !== null &&
+                    storedLocation.split(" ")[1] !== undefined &&
+                    storedLocation.split(" ")[1].startsWith("/training-material/")
+                ) {
+                    onloadscroll = storedLocation.split(" ")[0];
+                    url = storedLocation.split(" ")[1];
                 } else {
                     url = "/training-material/";
                 }
@@ -93,9 +96,9 @@ function addIframe() {
             });
 
             // Only setup the listener if it won't crash things.
-            if(safe) {
+            if (safe) {
                 // Listen to the scroll position
-                document.getElementById("gtn-embed").contentWindow.addEventListener('scroll', () => {
+                document.getElementById("gtn-embed").contentWindow.addEventListener("scroll", () => {
                     persistLocation();
                 });
             }
@@ -103,12 +106,12 @@ function addIframe() {
             // Depends on the iframe being present
             document.getElementById("gtn-embed").addEventListener("load", () => {
                 // Save our current location when possible
-                if(onloadscroll !== undefined){
-                    document.getElementById('gtn-embed').contentWindow.scrollTo(0, parseInt(onloadscroll));
+                if (onloadscroll !== undefined) {
+                    document.getElementById("gtn-embed").contentWindow.scrollTo(0, parseInt(onloadscroll));
                     onloadscroll = undefined;
                 }
 
-                if(safe) {
+                if (safe) {
                     persistLocation();
                 }
                 var gtn_tools = $("#gtn-embed").contents().find("span[data-tool]");