Merge branch 'dev' into ometiff2_new

lib/galaxy/app.py

@@ -2,7 +2,7 @@ import logging
 import signal
 import sys
 import time
-from typing import Any
+from typing import Any, Callable, List, Tuple
 
 from sqlalchemy.orm.scoping import (
     scoped_session,
@@ -70,19 +70,41 @@ from galaxy.web_stack import application_stack_instance, ApplicationStack
 from galaxy.webhooks import WebhooksRegistry
 from galaxy.workflow.trs_proxy import TrsProxy
 from .di import Container
-from .structured_app import BasicApp, StructuredApp
+from .structured_app import BasicApp, MinimalManagerApp, StructuredApp
 
 log = logging.getLogger(__name__)
 app = None
 
 
-class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container):
-    """Encapsulates the state of a Universe application"""
+class HaltableContainer(Container):
+    haltables: List[Tuple[str, Callable]]
 
-    def __init__(self, **kwargs) -> None:
+    def __init__(self) -> None:
         super().__init__()
+        self.haltables = []
+
+    def shutdown(self):
+        exception = None
+        for what, haltable in self.haltables:
+            try:
+                haltable()
+            except Exception as e:
+                log.exception(f"Failed to shutdown {what} cleanly")
+                exception = exception or e
+        if exception is not None:
+            raise exception
+
+
+class MinimalGalaxyApplication(BasicApp, config.ConfiguresGalaxyMixin, HaltableContainer):
+    """Encapsulates the state of a minimal Galaxy application"""
+
+    def __init__(self, fsmon=False, configure_logging=True, **kwargs) -> None:
+        super().__init__()
+        self.haltables = [
+            ("object store", self._shutdown_object_store),
+            ("database connection", self._shutdown_model),
+        ]
         self._register_singleton(BasicApp, self)
-        self._register_singleton(StructuredApp, self)
         if not log.handlers:
             # Paste didn't handle it, so we need a temporary basic log
             # configured.  The handler added here gets dumped and replaced with
@@ -91,38 +113,47 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container
         log.debug("python path is: %s", ", ".join(sys.path))
         self.name = 'galaxy'
         self.is_webapp = False
-        startup_timer = ExecutionTimer()
         self.new_installation = False
         # Read config file and check for errors
         self.config: Any = self._register_singleton(config.Configuration, config.Configuration(**kwargs))
         self.config.check()
-        config.configure_logging(self.config)
-        self.execution_timer_factory = self._register_singleton(ExecutionTimerFactory, ExecutionTimerFactory(self.config))
-        self.configure_fluent_log()
-        # A lot of postfork initialization depends on the server name, ensure it is set immediately after forking before other postfork functions
-        self.application_stack = self._register_singleton(ApplicationStack, application_stack_instance(app=self))
-        self.application_stack.register_postfork_function(self.application_stack.set_postfork_server_name, self)
-        self.config.reload_sanitize_allowlist(explicit='sanitize_allowlist_file' in kwargs)
-        self.amqp_internal_connection_obj = galaxy.queues.connection_from_config(self.config)
-        # queue_worker *can* be initialized with a queue, but here we don't
-        # want to and we'll allow postfork to bind and start it.
-        self.queue_worker = self._register_singleton(GalaxyQueueWorker, GalaxyQueueWorker(self))
-
-        self._configure_tool_shed_registry()
+        if configure_logging:
+            config.configure_logging(self.config)
         self._configure_object_store(fsmon=True)
-        # Setup the database engine and ORM
         config_file = kwargs.get('global_conf', {}).get('__file__', None)
         if config_file:
             log.debug('Using "galaxy.ini" config file: %s', config_file)
         check_migrate_tools = self.config.check_migrate_tools
         self._configure_models(check_migrate_databases=self.config.check_migrate_databases, check_migrate_tools=check_migrate_tools, config_file=config_file)
-
         # Security helper
         self._configure_security()
         self._register_singleton(IdEncodingHelper, self.security)
         self._register_singleton(SharedModelMapping, self.model)
         self._register_singleton(GalaxyModelMapping, self.model)
         self._register_singleton(scoped_session, self.model.context)
+
+    def configure_fluent_log(self):
+        if self.config.fluent_log:
+            from galaxy.util.custom_logging.fluent_log import FluentTraceLogger
+            self.trace_logger = FluentTraceLogger('galaxy', self.config.fluent_host, self.config.fluent_port)
+        else:
+            self.trace_logger = None
+
+    def _shutdown_object_store(self):
+        self.object_store.shutdown()
+
+    def _shutdown_model(self):
+        self.model.engine.dispose()
+
+
+class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication):
+    """Extends the MinimalGalaxyApplication with most managers that are not tied to a web or job handling context."""
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._register_singleton(MinimalManagerApp, self)
+        self.execution_timer_factory = self._register_singleton(ExecutionTimerFactory, ExecutionTimerFactory(self.config))
+        self.configure_fluent_log()
+
         # Tag handler
         self.tag_handler = self._register_singleton(GalaxyTagHandler)
         self.user_manager = self._register_singleton(UserManager)
@@ -133,17 +164,66 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container
         self.dataset_collections_service = self._register_singleton(DatasetCollectionManager)
         self.workflow_manager = self._register_singleton(WorkflowsManager)
         self.workflow_contents_manager = self._register_singleton(WorkflowContentsManager)
-        self.dependency_resolvers_view = self._register_singleton(DependencyResolversView, DependencyResolversView(self))
-        self.test_data_resolver = self._register_singleton(TestDataResolver, TestDataResolver(file_dirs=self.config.tool_test_data_directories))
         self.library_folder_manager = self._register_singleton(FolderManager)
         self.library_manager = self._register_singleton(LibraryManager)
         self.role_manager = self._register_singleton(RoleManager)
-        self.dynamic_tool_manager = self._register_singleton(DynamicToolManager)
-        self.api_keys_manager = self._register_singleton(ApiKeyManager)
 
         # ConfiguredFileSources
         self.file_sources = self._register_singleton(ConfiguredFileSources, ConfiguredFileSources.from_app_config(self.config))
 
+        # We need the datatype registry for running certain tasks that modify HDAs, and to build the registry we need
+        # to setup the installed repositories ... this is not ideal
+        self._configure_tool_config_files()
+        self.installed_repository_manager = self._register_singleton(InstalledRepositoryManager, InstalledRepositoryManager(self))
+        self._configure_datatypes_registry(self.installed_repository_manager)
+        self._register_singleton(Registry, self.datatypes_registry)
+        galaxy.model.set_datatypes_registry(self.datatypes_registry)
+
+        self.sentry_client = None
+        if self.config.sentry_dsn:
+
+            def postfork_sentry_client():
+                import raven
+                self.sentry_client = raven.Client(self.config.sentry_dsn, transport=raven.transport.HTTPTransport)
+
+            self.application_stack.register_postfork_function(postfork_sentry_client)
+
+
+class UniverseApplication(StructuredApp, GalaxyManagerApplication):
+    """Encapsulates the state of a Universe application"""
+
+    def __init__(self, **kwargs) -> None:
+        startup_timer = ExecutionTimer()
+        super().__init__(fsmon=True, **kwargs)
+        self.haltables = [
+            ("queue worker", self._shutdown_queue_worker),
+            ("file watcher", self._shutdown_watcher),
+            ("database heartbeat", self._shutdown_database_heartbeat),
+            ("workflow scheduler", self._shutdown_scheduling_manager),
+            ("object store", self._shutdown_object_store),
+            ("job manager", self._shutdown_job_manager),
+            ("application heartbeat", self._shutdown_heartbeat),
+            ("repository manager", self._shutdown_repo_manager),
+            ("database connection", self._shutdown_model),
+            ("application stack", self._shutdown_application_stack),
+        ]
+        self._register_singleton(StructuredApp, self)
+        # A lot of postfork initialization depends on the server name, ensure it is set immediately after forking before other postfork functions
+        self.application_stack = self._register_singleton(ApplicationStack, application_stack_instance(app=self))
+        self.application_stack.register_postfork_function(self.application_stack.set_postfork_server_name, self)
+        self.config.reload_sanitize_allowlist(explicit='sanitize_allowlist_file' in kwargs)
+        self.amqp_internal_connection_obj = galaxy.queues.connection_from_config(self.config)
+        # queue_worker *can* be initialized with a queue, but here we don't
+        # want to and we'll allow postfork to bind and start it.
+        self.queue_worker = self._register_singleton(GalaxyQueueWorker, GalaxyQueueWorker(self))
+
+        self._configure_tool_shed_registry()
+
+        self.dependency_resolvers_view = self._register_singleton(DependencyResolversView, DependencyResolversView(self))
+        self.test_data_resolver = self._register_singleton(TestDataResolver, TestDataResolver(file_dirs=self.config.tool_test_data_directories))
+        self.dynamic_tool_manager = self._register_singleton(DynamicToolManager)
+        self.api_keys_manager = self._register_singleton(ApiKeyManager)
+
         # Tool Data Tables
         self._configure_tool_data_tables(from_shed_config=False)
         # Load dbkey / genome build manager
@@ -169,14 +249,7 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container
         self.tool_shed_repository_cache = self._register_singleton(ToolShedRepositoryCache)
         # Watch various config files for immediate reload
         self.watchers = self._register_singleton(ConfigWatchers)
-        self._configure_tool_config_files()
-        self.installed_repository_manager = self._register_singleton(InstalledRepositoryManager, InstalledRepositoryManager(self))
-        self._configure_datatypes_registry(self.installed_repository_manager)
-        self._register_singleton(Registry, self.datatypes_registry)
-        galaxy.model.set_datatypes_registry(self.datatypes_registry)
-
         self._configure_toolbox()
-
         # Load Data Manager
         self.data_managers = self._register_singleton(DataManagers)
         # Load the update repository manager.
@@ -231,16 +304,6 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container
             self.authnz_manager = managers.AuthnzManager(self,
                                                          self.config.oidc_config_file,
                                                          self.config.oidc_backends_config_file)
-
-        self.sentry_client = None
-        if self.config.sentry_dsn:
-
-            def postfork_sentry_client():
-                import raven
-                self.sentry_client = raven.Client(self.config.sentry_dsn, transport=raven.transport.HTTPTransport)
-
-            self.application_stack.register_postfork_function(postfork_sentry_client)
-
         # Start the job manager
         from galaxy.jobs import manager
         self.job_manager = self._register_singleton(manager.JobManager)
@@ -284,8 +347,6 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container
         # Delay toolbox index until after startup
         self.application_stack.register_postfork_function(lambda: send_local_control_task(self, 'rebuild_toolbox_search_index'))
 
-        self.model.engine.dispose()
-
         # Inject url_for for components to more easily optionally depend
         # on url_for.
         self.url_for = url_for
@@ -293,74 +354,30 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container
         self.server_starttime = int(time.time())  # used for cachebusting
         log.info("Galaxy app startup finished %s" % startup_timer)
 
-    def shutdown(self):
-        log.debug('Shutting down')
-        exception = None
-        try:
-            self.queue_worker.shutdown()
-        except Exception as e:
-            exception = exception or e
-            log.exception("Failed to shutdown control worker cleanly")
-        try:
-            self.watchers.shutdown()
-        except Exception as e:
-            exception = exception or e
-            log.exception("Failed to shutdown configuration watchers cleanly")
-        try:
-            self.database_heartbeat.shutdown()
-        except Exception as e:
-            exception = exception or e
-            log.exception("Failed to shutdown database heartbeat cleanly")
-        try:
-            self.workflow_scheduling_manager.shutdown()
-        except Exception as e:
-            exception = exception or e
-            log.exception("Failed to shutdown workflow scheduling manager cleanly")
-        try:
-            self.job_manager.shutdown()
-        except Exception as e:
-            exception = exception or e
-            log.exception("Failed to shutdown job manager cleanly")
-        try:
-            self.object_store.shutdown()
-        except Exception as e:
-            exception = exception or e
-            log.exception("Failed to shutdown object store cleanly")
-        try:
-            if self.heartbeat:
-                self.heartbeat.shutdown()
-        except Exception as e:
-            exception = exception or e
-            log.exception("Failed to shutdown heartbeat cleanly")
-        try:
-            self.update_repository_manager.shutdown()
-        except Exception as e:
-            exception = exception or e
-            log.exception("Failed to shutdown update repository manager cleanly")
+    def _shutdown_queue_worker(self):
+        self.queue_worker.shutdown()
 
-        try:
-            self.model.engine.dispose()
-        except Exception as e:
-            exception = exception or e
-            log.exception("Failed to shutdown SA database engine cleanly")
+    def _shutdown_watcher(self):
+        self.watchers.shutdown()
 
-        try:
-            self.application_stack.shutdown()
-        except Exception as e:
-            exception = exception or e
-            log.exception("Failed to shutdown application stack interface cleanly")
+    def _shutdown_database_heartbeat(self):
+        self.database_heartbeat.shutdown()
 
-        if exception:
-            raise exception
-        else:
-            log.debug('Finished shutting down')
+    def _shutdown_scheduling_manager(self):
+        self.workflow_scheduling_manager.shutdown()
 
-    def configure_fluent_log(self):
-        if self.config.fluent_log:
-            from galaxy.util.custom_logging.fluent_log import FluentTraceLogger
-            self.trace_logger = FluentTraceLogger('galaxy', self.config.fluent_host, self.config.fluent_port)
-        else:
-            self.trace_logger = None
+    def _shutdown_job_manager(self):
+        self.job_manager.shutdown()
+
+    def _shutdown_heartbeat(self):
+        if self.heartbeat:
+            self.heartbeat.shutdown()
+
+    def _shutdown_repo_manager(self):
+        self.update_repository_manager.shutdown()
+
+    def _shutdown_application_stack(self):
+        self.application_stack.shutdown()
 
     @property
     def is_job_handler(self) -> bool:

lib/galaxy/celery/__init__.py

@@ -0,0 +1,55 @@
+import os
+from functools import lru_cache
+
+from celery import Celery
+
+from galaxy.config import Configuration
+from galaxy.util.custom_logging import get_logger
+from galaxy.util.properties import load_app_properties
+
+log = get_logger(__name__)
+
+
+@lru_cache(maxsize=1)
+def get_galaxy_app():
+    import galaxy.app
+    if galaxy.app.app:
+        return galaxy.app.app
+    kwargs = get_app_properties()
+    if kwargs:
+        kwargs['check_migrate_tools'] = False
+        kwargs['check_migrate_databases'] = False
+        galaxy_app = galaxy.app.GalaxyManagerApplication(configure_logging=False, **kwargs)
+        return galaxy_app
+
+
+@lru_cache(maxsize=1)
+def get_app_properties():
+    config_file = os.environ.get("GALAXY_CONFIG_FILE")
+    if config_file:
+        return load_app_properties(
+            config_file=os.path.abspath(config_file),
+            config_section='galaxy',
+        )
+
+
+@lru_cache(maxsize=1)
+def get_config():
+    kwargs = get_app_properties()
+    if kwargs:
+        kwargs['override_tempdir'] = False
+        return Configuration(**kwargs)
+
+
+def get_broker():
+    config = get_config()
+    if config:
+        return config.amqp_internal_connection
+
+
+broker = get_broker()
+celery_app = Celery('galaxy', broker=broker, include=['galaxy.celery.tasks'])
+
+
+if __name__ == '__main__':
+    celery_app.start()

lib/galaxy/celery/tasks.py

@@ -0,0 +1,51 @@
+from lagom import magic_bind_to_container
+from sqlalchemy.orm.scoping import (
+    scoped_session,
+)
+
+from galaxy.celery import celery_app
+from galaxy.managers.hdas import HDAManager
+from galaxy.managers.lddas import LDDAManager
+from galaxy.model import User
+from galaxy.util.custom_logging import get_logger
+from . import get_galaxy_app
+
+log = get_logger(__name__)
+
+
+def galaxy_task(func):
+    app = get_galaxy_app()
+    if app:
+        return magic_bind_to_container(app)(func)
+    return func
+
+
+@celery_app.task(ignore_result=True)
+@galaxy_task
+def recalculate_user_disk_usage(session: scoped_session, user_id=None):
+    if user_id:
+        user = session.query(User).get(user_id)
+        if user:
+            user.calculate_and_set_disk_usage()
+            log.info(f"New user disk usage is {user.disk_usage}")
+        else:
+            log.error("Recalculate user disk usage task failed, user %s not found" % user_id)
+    else:
+        log.error("Recalculate user disk usage task received without user_id.")
+
+
+@celery_app.task(ignore_result=True)
+@galaxy_task
+def purge_hda(hda_manager: HDAManager, hda_id):
+    hda = hda_manager.by_id(hda_id)
+    hda_manager._purge(hda)
+
+
+@celery_app.task
+@galaxy_task
+def set_metadata(hda_manager: HDAManager, ldda_manager: LDDAManager, dataset_id, model_class='HistoryDatasetAssociation'):
+    if model_class == 'HistoryDatasetAssociation':
+        dataset = hda_manager.by_id(dataset_id)
+    elif model_class == 'LibraryDatasetDatasetAssociation':
+        dataset = ldda_manager.by_id(dataset_id)
+    dataset.datatype.set_meta(dataset)

lib/galaxy/config/__init__.py

@@ -29,6 +29,7 @@ from galaxy.config.schema import AppSchema
 from galaxy.containers import parse_containers_config
 from galaxy.exceptions import ConfigurationError
 from galaxy.model import mapping
+from galaxy.model.database_utils import database_exists
 from galaxy.model.tool_shed_install.migrate.check import create_or_verify_database as tsi_create_or_verify_database
 from galaxy.util import (
     ExecutionTimer,
@@ -124,7 +125,7 @@ class BaseAppConfiguration:
     listify_options: Set[str] = set()  # values for these options are processed as lists of values
 
     def __init__(self, **kwargs):
-        self._process_renamed_options(kwargs)
+        self._preprocess_kwargs(kwargs)
         self._kwargs = kwargs  # Save these as a record of explicitly set options
         self.config_dict = kwargs
         self.root = find_root(kwargs)
@@ -137,6 +138,10 @@ class BaseAppConfiguration:
         self._resolve_paths()  # Overwrite attribute values with resolved paths
         self._postprocess_paths_to_resolve()  # Any steps that need to happen after paths are resolved
 
+    def _preprocess_kwargs(self, kwargs):
+        self._process_renamed_options(kwargs)
+        self._fix_postgresql_dburl(kwargs)
+
     def _process_renamed_options(self, kwargs):
         """Update kwargs to set any unset renamed options to values of old-named options, if set.
 
@@ -147,6 +152,25 @@ class BaseAppConfiguration:
                 if old in kwargs and new not in kwargs:
                     kwargs[new] = kwargs[old]
 
+    def _fix_postgresql_dburl(self, kwargs):
+        """
+        Fix deprecated database URLs (postgres... >> postgresql...)
+        https://docs.sqlalchemy.org/en/14/changelog/changelog_14.html#change-3687655465c25a39b968b4f5f6e9170b
+        """
+        old_dialect, new_dialect = 'postgres', 'postgresql'
+        old_prefixes = (f'{old_dialect}:', f'{old_dialect}+')  # check for postgres://foo and postgres+driver//foo
+        offset = len(old_dialect)
+        keys = ('database_connection', 'install_database_connection')
+        for key in keys:
+            if key in kwargs:
+                value = kwargs[key]
+                for prefix in old_prefixes:
+                    if value.startswith(prefix):
+                        value = f'{new_dialect}{value[offset:]}'
+                        kwargs[key] = value
+                        log.warning('PostgreSQL database URLs of the form "postgres://" have been '
+                            'deprecated. Please use "postgresql://".')
+
     def is_set(self, key):
         """Check if a configuration option has been explicitly set."""
         # NOTE: This will check all supplied keyword arguments, including those not in the schema.
@@ -530,7 +554,11 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
         self._process_config(kwargs)
 
     def _load_schema(self):
-        return AppSchema(GALAXY_CONFIG_SCHEMA_PATH, GALAXY_APP_NAME)
+        # Schemas are symlinked to the root of the galaxy-app package
+        config_schema_path = os.path.join(os.path.dirname(__file__), os.pardir, 'config_schema.yml')
+        if os.path.exists(GALAXY_CONFIG_SCHEMA_PATH):
+            config_schema_path = GALAXY_CONFIG_SCHEMA_PATH
+        return AppSchema(config_schema_path, GALAXY_APP_NAME)
 
     def _override_tempdir(self, kwargs):
         if string_as_bool(kwargs.get("override_tempdir", "True")):
@@ -1270,7 +1298,6 @@ class ConfiguresGalaxyMixin:
             signal.signal(sig, handler)
 
     def _wait_for_database(self, url):
-        from sqlalchemy_utils import database_exists
         attempts = self.config.database_wait_attempts
         pause = self.config.database_wait_sleep
         for i in range(1, attempts):

lib/galaxy/config/config_manage.py

@@ -351,7 +351,7 @@ GALAXY_APP = App(
 SHED_APP = App(
     ["tool_shed_wsgi.ini", "config/tool_shed.ini"],
     "9009",
-    ["galaxy.webapps.tool_shed.buildapp:app_factory"],
+    ["tool_shed.webapp.buildapp:app_factory"],
     "config/tool_shed.yml",
     "lib/tool_shed/webapp/config_schema.yml",
     'tool_shed.webapp.buildapp:uwsgi_app()',

lib/galaxy/config/sample/datatypes_conf.xml.sample

@@ -197,6 +197,7 @@
     <datatype extension="gatk_dbsnp" type="galaxy.datatypes.tabular:Tabular" subclass="true" display_in_upload="true"/>
     <datatype extension="gatk_tranche" type="galaxy.datatypes.tabular:Tabular" subclass="true" display_in_upload="true"/>
     <datatype extension="gatk_recal" type="galaxy.datatypes.tabular:Tabular" subclass="true" display_in_upload="true"/>
+    <datatype extension="hhr" type="galaxy.datatypes.data:Text" subclass="true" display_in_upload="true"/>
     <datatype extension="jpg" type="galaxy.datatypes.images:Jpg" mimetype="image/jpeg"/>
     <datatype extension="tiff" type="galaxy.datatypes.images:Tiff" mimetype="image/tiff" display_in_upload="true"/>
     <datatype extension="tf2" type="galaxy.datatypes.images:Tiff" subclass="true" display_in_upload="false"/>
@@ -267,6 +268,7 @@
     <datatype extension="mztab2" type="galaxy.datatypes.proteomics:MzTab2" display_in_upload="true"/>
     <datatype extension="mzml" type="galaxy.datatypes.proteomics:MzML" mimetype="application/xml" display_in_upload="true"/>
     <datatype extension="nmrml" type="galaxy.datatypes.proteomics:NmrML" mimetype="application/xml" display_in_upload="true" description="nmrML is an open mark-up language for NMR data." description_url="http://nmrml.org/schema/"/>
+    <datatype extension="meryldb" type="galaxy.datatypes.binary:Meryldb" subclass="true" display_in_upload="true" description="MerylDB is a tar.gz archive containing 64 binaries + 64 indexes." />
     <datatype extension="mgf" type="galaxy.datatypes.proteomics:Mgf" display_in_upload="true"/>
     <datatype extension="wiff" type="galaxy.datatypes.proteomics:Wiff" display_in_upload="true"/>
     <datatype extension="mzxml" type="galaxy.datatypes.proteomics:MzXML" mimetype="application/xml" display_in_upload="true"/>
@@ -313,6 +315,7 @@
     <datatype extension="gii" auto_compressed_types="gz" type="galaxy.datatypes.images:Gifti" mimetype="application/octet-stream" display_in_upload="true" />
     <datatype extension="tck" type="galaxy.datatypes.images:Tck" mimetype="application/octet-stream" display_in_upload="true" />
     <datatype extension="trk" type="galaxy.datatypes.images:Trk" mimetype="application/octet-stream" display_in_upload="true" />
+    <datatype extension="mrc" type="galaxy.datatypes.images:Mrc2014" mimetype="application/octet-stream" display_in_upload="true" />
     <datatype extension="peff" type="galaxy.datatypes.proteomics:PEFF" display_in_upload="true" />
     <!-- End Proteomics Datatypes -->
     <!-- FlowCytometry -->
@@ -852,6 +855,7 @@
     <datatype extension="par" type="galaxy.datatypes.speech:BPF" display_in_upload="true" mimetype="text/plain-bas"/>
     <datatype extension="ffindex" type="galaxy.datatypes.tabular:Tabular" display_in_upload="true" subclass="true" description_url="https://github.com/soedinglab/ffindex_soedinglab"/>
     <datatype extension="ffdata" type="galaxy.datatypes.data:Data" display_in_upload="true" subclass="true" description_url="https://github.com/soedinglab/ffindex_soedinglab"/>
+    <datatype extension="pretext" type="galaxy.datatypes.binary:Pretext" display_in_upload="true" />
   </registration>
   <sniffers>
     <!--
@@ -923,6 +927,7 @@
     <sniffer type="galaxy.datatypes.binary:Fast5ArchiveGz"/>
     <sniffer type="galaxy.datatypes.binary:Fast5ArchiveBz2"/>
     <sniffer type="galaxy.datatypes.binary:Fast5Archive"/>
+    <sniffer type="galaxy.datatypes.binary:Meryldb" />
     <sniffer type="galaxy.datatypes.binary:PostgresqlArchive"/>
     <sniffer type="galaxy.datatypes.binary:ICM"/>
     <sniffer type="galaxy.datatypes.binary:Idat"/>
@@ -934,6 +939,7 @@
     <sniffer type="galaxy.datatypes.binary:Vel"/>
     <sniffer type="galaxy.datatypes.binary:Xlsx"/>
     <sniffer type="galaxy.datatypes.binary:CompressedZipArchive"/>
+    <sniffer type="galaxy.datatypes.binary:Pretext"/>
     <sniffer type="galaxy.datatypes.annotation:Augustus"/>
     <sniffer type="galaxy.datatypes.triples:Rdf"/>
     <sniffer type="galaxy.datatypes.blast:BlastXml"/>
@@ -1048,6 +1054,7 @@
     <sniffer type="galaxy.datatypes.microarrays:Gpr"/>
     <sniffer type="galaxy.datatypes.microarrays:Gal"/>
     <sniffer type="galaxy.datatypes.binary:RData"/>
+    <sniffer type="galaxy.datatypes.images:Mrc2014"/>
     <sniffer type="galaxy.datatypes.images:Jpg"/>
     <sniffer type="galaxy.datatypes.images:Png"/>
     <sniffer type="galaxy.datatypes.images:OMETiff"/>

lib/galaxy/config/sample/file_sources_conf.yml.sample

@@ -18,3 +18,19 @@
   label: Posix
   doc: Files from local path
   root: /some/path/
+
+- type: s3fs
+  label: My MinIO storage
+  endpoint_url: "https://minio.usegalaxy.eu"
+  id: galaxy-minio-storage
+  doc: Galaxy MinIO S3 storage
+  anon: false
+  secret: "UHAJ6asd6asdhasd"
+  key: "MCJU76agdt98GGFAROIP7"
+
+- type: s3fs
+  label: CMIP6 GCMs downscaled using WRF
+  id: wrf-cmip6-noversioning
+  doc: High-resolution historical and future climate simulations from 1980-2100
+  bucket: wrf-cmip6-noversioning
+  anon: true

lib/galaxy/config/sample/galaxy.yml.sample

@@ -1684,6 +1684,13 @@ galaxy:
   # disable any such maximum.
   #maximum_workflow_jobs_per_scheduling_iteration: 1000
 
+  # Maximum number of datasets to create before flushing created
+  # datasets to database. This affects tools that create many output
+  # datasets. Higher values will lead to fewer database flushes and
+  # faster execution, but require more memory. Set to -1 to disable
+  # creating datasets in batches.
+  #flush_per_n_datasets: 1000
+
   # Force serial scheduling of workflows within the context of a
   # particular history
   #history_local_serial_workflow_scheduling: false
@@ -2152,6 +2159,11 @@ galaxy:
   # commented out line below).
   #amqp_internal_connection: sqlalchemy+sqlite:///./database/control.sqlite?isolation_level=IMMEDIATE
 
+  # Offload long-running tasks to a Celery task queue. Activate this
+  # only if you have setup a Celery worker for Galaxy. For details, see
+  # https://docs.galaxyproject.org/en/master/admin/production.html
+  #enable_celery_tasks: false
+
   # Allow disabling pbkdf2 hashing of passwords for legacy situations.
   # This should normally be left enabled unless there is a specific
   # reason to disable it.

lib/galaxy/config/sample/user_preferences_extra_conf.yml.sample

@@ -59,10 +59,11 @@ preferences:
               required: False
               options:
                   - [Navigator default, auto]
-                  - [Chinese, zh]
+                  - [中文, zh]
                   - [English, en]
-                  - [French, fr]
-                  - [Japanese, ja]
+                  - [Español, es]
+                  - [Français, fr]
+                  - [日本語, ja]
 
     # Used in file_sources_conf.yml
     dropbox:

lib/galaxy/datatypes/binary.py

@@ -201,6 +201,34 @@ class CompressedArchive(Binary):
             return "Compressed binary file (%s)" % (nice_size(dataset.get_size()))
 
 
+class Meryldb(CompressedArchive):
+    """MerylDB is a tar.gz archive, with 128 files. 64 data files and 64 index files."""
+    file_ext = "meryldb"
+
+    def sniff(self, filename):
+        """
+        Try to guess if the file is a Cel file.
+        >>> from galaxy.datatypes.sniff import get_test_fname
+        >>> fname = get_test_fname('affy_v_agcc.cel')
+        >>> Meryldb().sniff(fname)
+        False
+        >>> fname = get_test_fname('read-db.meryldb')
+        >>> Meryldb().sniff(fname)
+        True
+        """
+        try:
+            if filename and tarfile.is_tarfile(filename):
+                with tarfile.open(filename, 'r') as temptar:
+                    _tar_content = temptar.getnames()
+                    # 64 data files ad 64 indices + 2 folders
+                    if len(_tar_content) == 130:
+                        if len([_ for _ in _tar_content if _.endswith('.merylIndex')]) == 64:
+                            return True
+        except Exception as e:
+            log.warning('%s, sniff Exception: %s', self, e)
+        return False
+
+
 class DynamicCompressedArchive(CompressedArchive):
 
     def matches_any(self, target_datatypes):
@@ -2903,6 +2931,37 @@ class WiffTar(BafTar):
         return "Sciex WIFF/SCAN archive"
 
 
+@build_sniff_from_prefix
+class Pretext(Binary):
+    """
+    PretextMap contact map file
+    Try to guess if the file is a Pretext file.
+    >>> from galaxy.datatypes.sniff import get_test_fname
+    >>> fname = get_test_fname('sample.pretext')
+    >>> Pretext().sniff(fname)
+    True
+    """
+
+    def sniff_prefix(self, sniff_prefix):
+        # The first 4 bytes of any pretext file is 'pstm', and the rest of the
+        # file contains binary data.
+        return sniff_prefix.startswith_bytes(b'pstm')
+
+    def set_peek(self, dataset, is_multi_byte=False):
+        if not dataset.dataset.purged:
+            dataset.peek = "Binary pretext file"
+            dataset.blurb = nice_size(dataset.get_size())
+        else:
+            dataset.peek = 'file does not exist'
+            dataset.blurb = 'file purged from disk'
+
+    def display_peek(self, dataset):
+        try:
+            return dataset.peek
+        except Exception:
+            return "Binary pretext file (%s)" % (nice_size(dataset.get_size()))
+
+
 if __name__ == '__main__':
     import doctest
     doctest.testmod(sys.modules[__name__])

lib/galaxy/datatypes/display_applications/configs/biom/biom_simple.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0"?>
+<display id="biom_simple" version="1.0.0" name="view biom at">
+    <dynamic_links from_data_table="biom_simple_display" skip_startswith="#" id="value" name="name">
+        <url>${ url % { 'biom_file_url_qp': $biom_file.qp } }</url>
+        <param type="data" name="biom_file" url="galaxy_${DATASET_HASH}.biom" />
+    </dynamic_links>
+</display>

lib/galaxy/datatypes/display_applications/configs/ensembl/ensembl_bam.xml

@@ -0,0 +1,21 @@
+<display id="ensembl_bam" version="1.0.0" name="display at Ensembl">
+    <!-- Current Ensembl method of attaching user data via URL; archives older than ~November 2008 will use a different method -->
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="ensembl" skip_startswith="#" id="0" name="1">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="site_name" value="1"/>
+        <dynamic_param name="site_link" value="2"/>
+        <dynamic_param name="site_dbkeys" value="3" split="True" separator="," />
+        <dynamic_param name="site_organisms" value="4" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_dbkeys to dataset dbkey -->
+        <filter>${dataset.dbkey in $site_dbkeys}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${site_link}${site_organism}/Location/View?contigviewbottom=bam:${bam_file.qp}=normal</url>
+        <param type="data" name="bam_file" url="galaxy_${DATASET_HASH}.bam" strip_https="True" />
+        <param type="data" name="bai_file" url="galaxy_${DATASET_HASH}.bam.bai" metadata="bam_index" strip_https="True" />
+        <param type="template" name="site_organism" strip="True" >
+            $site_organisms[ $site_dbkeys.index( $bam_file.dbkey ) ]
+        </param>
+    </dynamic_links>
+</display>

lib/galaxy/datatypes/display_applications/configs/ensembl/ensembl_gff.xml

@@ -0,0 +1,69 @@
+<display id="ensembl_gff" version="1.0.0" name="display at Ensembl">
+    <!-- Current Ensembl method of attaching user data via URL; archives older than ~November 2008 will use a different method -->
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="ensembl" skip_startswith="#" id="0" name="1">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="site_name" value="1"/>
+        <dynamic_param name="site_link" value="2"/>
+        <dynamic_param name="site_dbkeys" value="3" split="True" separator="," />
+        <dynamic_param name="site_organisms" value="4" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_dbkeys to dataset dbkey -->
+        <filter>${dataset.dbkey in $site_dbkeys}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${site_link}${site_organism}/Location/View?r=${position};contigviewbottom=url:${gff_file.qp}=normal</url>
+        <param type="data" name="gff_file" url="galaxy_${DATASET_HASH}.gff" />
+        <param type="template" name="site_organism" strip="True" >
+            $site_organisms[ $site_dbkeys.index( $gff_file.dbkey ) ]
+        </param>
+        <param type="template" name="position" strip="True" >
+#set chrom, start, end = $gff_file.datatype.get_estimated_display_viewport( $gff_file )
+#if $chrom is not None:
+##The difference between chr1 and 1 is handled by Ensembl, except for the viewport, we need to provide e.g. '1' instead of 'chr1' here
+##This is rather naive, it would be more ideal to have actual mappings
+#if $chrom == 'chrM':
+    #set $chrom = 'MT'
+#end if
+#if $chrom.startswith( 'chr' ):
+    #set $chrom = $chrom[3:]
+#end if
+${chrom}:${start}-${end}
+#else:
+##default view is of '1'
+1
+#end if
+        </param>
+    </dynamic_links>
+    <!-- Old Ensembl method of attaching user data via URL -->
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="ensembl_data_url" skip_startswith="#" id="0" name="1">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="site_name" value="1"/>
+        <dynamic_param name="site_link" value="2"/>
+        <dynamic_param name="site_dbkeys" value="3" split="True" separator="," />
+        <dynamic_param name="site_organisms" value="4" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_dbkeys to dataset dbkey -->
+        <filter>${dataset.dbkey in $site_dbkeys}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${site_link}${site_organism}/contigview?data_URL=${gff_file.qp}${position}</url>
+        <param type="data" name="gff_file" url="galaxy_${DATASET_HASH}.gff" />
+        <param type="template" name="site_organism" strip="True" >
+            $site_organisms[ $site_dbkeys.index( $gff_file.dbkey ) ]
+        </param>
+        <param type="template" name="position" strip="True" >
+            #set chrom, start, end = $gff_file.datatype.get_estimated_display_viewport( $gff_file )
+            #if $chrom is not None:
+            ##The difference between chr1 and 1 is handled by Ensembl, except for the viewport, we need to provide e.g. '1' instead of 'chr1' here
+            ##This is rather naive, it would be more ideal to have actual mappings
+            #if $chrom == 'chrM':
+                #set $chrom = 'MT'
+            #end if
+            #if $chrom.startswith( 'chr' ):
+                #set $chrom = $chrom[3:]
+            #end if
+            &amp;chr=${chrom}&amp;start=${start}&amp;end=${end}
+            #end if
+        </param>
+    </dynamic_links>
+</display>

lib/galaxy/datatypes/display_applications/configs/ensembl/ensembl_interval_as_bed.xml

@@ -0,0 +1,69 @@
+<display id="ensembl_interval" version="1.0.0" name="display at Ensembl">
+    <!-- Current Ensembl method of attaching user data via URL; archives older than ~November 2008 will use a different method -->
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="ensembl" skip_startswith="#" id="0" name="1">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="site_name" value="1"/>
+        <dynamic_param name="site_link" value="2"/>
+        <dynamic_param name="site_dbkeys" value="3" split="True" separator="," />
+        <dynamic_param name="site_organisms" value="4" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_dbkeys to dataset dbkey -->
+        <filter>${dataset.dbkey in $site_dbkeys}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${site_link}${site_organism}/Location/View?r=${position};contigviewbottom=url:${bed_file.qp}=normal</url>
+        <param type="data" name="bed_file" url="galaxy_${DATASET_HASH}.bed" format="bedstrict"/>
+        <param type="template" name="site_organism" strip="True" >
+            $site_organisms[ $site_dbkeys.index( $bed_file.dbkey ) ]
+        </param>
+        <param type="template" name="position" strip="True" >
+#set chrom, start, end = $bed_file.datatype.get_estimated_display_viewport( $bed_file )
+#if $chrom is not None:
+##The difference between chr1 and 1 is handled by Ensembl, except for the viewport, we need to provide e.g. '1' instead of 'chr1' here
+##This is rather naive, it would be more ideal to have actual mappings
+#if $chrom == 'chrM':
+    #set $chrom = 'MT'
+#end if
+#if $chrom.startswith( 'chr' ):
+    #set $chrom = $chrom[3:]
+#end if
+${chrom}:${int( start ) + 1}-${end}
+#else:
+##default view is of '1'
+1
+#end if
+        </param>
+    </dynamic_links>
+    <!-- Old Ensembl method of attaching user data via URL -->
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="ensembl_data_url" skip_startswith="#" id="0" name="1">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="site_name" value="1"/>
+        <dynamic_param name="site_link" value="2"/>
+        <dynamic_param name="site_dbkeys" value="3" split="True" separator="," />
+        <dynamic_param name="site_organisms" value="4" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_dbkeys to dataset dbkey -->
+        <filter>${dataset.dbkey in $site_dbkeys}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${site_link}${site_organism}/contigview?data_URL=${bed_file.qp}${position}</url>
+        <param type="data" name="bed_file" url="galaxy_${DATASET_HASH}.bed" format="bedstrict"/>
+        <param type="template" name="site_organism" strip="True" >
+            $site_organisms[ $site_dbkeys.index( $bed_file.dbkey ) ]
+        </param>
+        <param type="template" name="position" strip="True" >
+            #set chrom, start, end = $bed_file.datatype.get_estimated_display_viewport( $bed_file )
+            #if $chrom is not None:
+            ##The difference between chr1 and 1 is handled by Ensembl, except for the viewport, we need to provide e.g. '1' instead of 'chr1' here
+            ##This is rather naive, it would be more ideal to have actual mappings
+            #if $chrom == 'chrM':
+                #set $chrom = 'MT'
+            #end if
+            #if $chrom.startswith( 'chr' ):
+                #set $chrom = $chrom[3:]
+            #end if
+            &amp;chr=${chrom}&amp;start=${int( start ) + 1}&amp;end=${end}
+            #end if
+        </param>
+    </dynamic_links>
+</display>

lib/galaxy/datatypes/display_applications/configs/gbrowse/gbrowse_gff.xml

@@ -0,0 +1,29 @@
+<display id="gbrowse_gff" version="1.0.0" name="display at GBrowse">
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="gbrowse" skip_startswith="#" id="0" name="1">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="site_name" value="1"/>
+        <dynamic_param name="site_link" value="2"/>
+        <dynamic_param name="site_dbkeys" value="3" split="True" separator="," />
+        <dynamic_param name="site_organisms" value="4" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_id to a Galaxy application configuration parameter and by dataset dbkey -->
+        <filter>${site_id in $APP.datatypes_registry.get_display_sites('gbrowse')}</filter>
+        <filter>${dataset.dbkey in $site_dbkeys}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${site_link}${site_organism}/?${position}eurl=${gff_file.qp}</url>
+        <param type="data" name="gff_file" url="galaxy_${DATASET_HASH}.gff" />
+        <param type="template" name="site_organism" strip="True" >
+            $site_organisms[ $site_dbkeys.index( $gff_file.dbkey ) ]
+        </param>
+        <param type="template" name="position" strip="True" >
+#set chrom, start, end = $gff_file.datatype.get_estimated_display_viewport( $gff_file )
+#if $chrom is not None:
+#if $chrom.startswith( 'chr' ):
+    #set $chrom = $chrom[3:]
+#end if
+q=${chrom}:${start}..${end}&amp;
+#end if
+        </param>
+    </dynamic_links>
+</display>

lib/galaxy/datatypes/display_applications/configs/gbrowse/gbrowse_interval_as_bed.xml

@@ -0,0 +1,29 @@
+<display id="gbrowse_interval_as_bed" version="1.0.0" name="display at GBrowse">
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="gbrowse" skip_startswith="#" id="0" name="1">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="site_name" value="1"/>
+        <dynamic_param name="site_link" value="2"/>
+        <dynamic_param name="site_dbkeys" value="3" split="True" separator="," />
+        <dynamic_param name="site_organisms" value="4" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_id to a Galaxy application configuration parameter and by dataset dbkey -->
+        <filter>${site_id in $APP.datatypes_registry.get_display_sites('gbrowse')}</filter>
+        <filter>${dataset.dbkey in $site_dbkeys}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${site_link}${site_organism}/?${position}eurl=${bed_file.qp}</url>
+        <param type="data" name="bed_file" url="galaxy_${DATASET_HASH}.bed" format="bedstrict"/> <!-- Galaxy allows BED files to contain non-standard fields beyond the first 3 columns, gbrowse does not(?): force use of converter which will make strict BED6+ file -->
+        <param type="template" name="site_organism" strip="True" >
+            $site_organisms[ $site_dbkeys.index( $bed_file.dbkey ) ]
+        </param>
+        <param type="template" name="position" strip="True" >
+#set chrom, start, end = $bed_file.datatype.get_estimated_display_viewport( $bed_file )
+#if $chrom is not None:
+#if $chrom.startswith( 'chr' ):
+    #set $chrom = $chrom[3:]
+#end if
+q=${chrom}:${start}..${end}&amp;
+#end if
+        </param>
+    </dynamic_links>
+</display>

lib/galaxy/datatypes/display_applications/configs/gbrowse/gbrowse_wig.xml

@@ -0,0 +1,29 @@
+<display id="gbrowse_wig" version="1.0.0" name="display at GBrowse">
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="gbrowse" skip_startswith="#" id="0" name="1">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="site_name" value="1"/>
+        <dynamic_param name="site_link" value="2"/>
+        <dynamic_param name="site_dbkeys" value="3" split="True" separator="," />
+        <dynamic_param name="site_organisms" value="4" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_id to a Galaxy application configuration parameter and by dataset dbkey -->
+        <filter>${site_id in $APP.datatypes_registry.get_display_sites('gbrowse')}</filter>
+        <filter>${dataset.dbkey in $site_dbkeys}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${site_link}${site_organism}/?${position}eurl=${wig_file.qp}</url>
+        <param type="data" name="wig_file" url="galaxy_${DATASET_HASH}.wig" format="wig"/>
+        <param type="template" name="site_organism" strip="True" >
+            $site_organisms[ $site_dbkeys.index( $wig_file.dbkey ) ]
+        </param>
+        <param type="template" name="position" strip="True" >
+#set chrom, start, end = $wig_file.datatype.get_estimated_display_viewport( $wig_file )
+#if $chrom is not None:
+#if $chrom.startswith( 'chr' ):
+    #set $chrom = $chrom[3:]
+#end if
+q=${chrom}:${start}..${end}&amp;
+#end if
+        </param>
+    </dynamic_links>
+</display>

lib/galaxy/datatypes/display_applications/configs/icn3d/icn3d_simple.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0"?>
+<display id="icn3d_simple" version="1.0.0" name="view in iCn3D at">
+    <dynamic_links from_data_table="icn3d_simple_display" skip_startswith="#" id="value" name="name">
+        <url>${ url % {  'icn3d_file_type': $icn3d_file.ext, 'icn3d_file_url_qp': $icn3d_file.qp } }</url>
+        <param type="data" name="icn3d_file" url="galaxy_${DATASET_HASH}.${dataset.ext}" allow_cors="true" />
+    </dynamic_links>
+</display>

lib/galaxy/datatypes/display_applications/configs/igb/bam.xml

@@ -0,0 +1,13 @@
+<display id="igb_bam" version="0.0.0" name="display in IGB">
+    <link id="View" name="View">
+        <url>http://bioviz.org/galaxy.html?version=${bam_file.dbkey}&amp;feature_url_0=${bam_file.url}&amp;sym_name_0=${niceName}&amp;sym_method_0=${bam_file.url}&amp;query_url=${bam_file.url}&amp;server_url=galaxy</url>
+        <param type="data" name="bam_file_for_name" viewable="False"/>
+        <param type="template" name="niceName" viewable="False" strip="True">
+            #import re
+            #set nm=$bam_file_for_name.name
+            ${re.sub('\W',"_",nm)}
+        </param>
+        <param type="data" name="bai_file" url="${niceName}.bam.bai" metadata="bam_index" />
+        <param type="data" name="bam_file" url="${niceName}.bam" />
+    </link>
+</display>

lib/galaxy/datatypes/display_applications/configs/igb/bb.xml

@@ -0,0 +1,12 @@
+<display id="igb_bb" version="1.0.0" name="display in IGB">
+    <link id="View" name="View">
+        <url>http://bioviz.org/galaxy.html?version=${bigbed_file.dbkey}&amp;loadresidues=false&amp;feature_url_0=${bigbed_file.url}&amp;sym_name_0=${niceName}&amp;sym_method_0=${bigbed_file.url}&amp;query_url=${bigbed_file.url}&amp;server_url=galaxy</url>
+        <param type="data" name="bigbed_file_for_name" viewable="False"/>
+        <param type="template" name="niceName" viewable="False" strip="True">
+            #import re
+            #set nm=$bigbed_file_for_name.name
+            ${re.sub('\W',"_",nm)}
+        </param>
+        <param type="data" name="bigbed_file" url="${niceName}.bigbed" />
+    </link>
+</display>

lib/galaxy/datatypes/display_applications/configs/igb/bed.xml

@@ -0,0 +1,15 @@
+<display id="igb_bed" version="1.0.0" name="display in IGB">
+    <link id="View" name="View">
+        <url>$bioviz</url>
+        <param type="data" name="bed_file_for_name" viewable="False"/>
+        <param type="template" name="niceName" viewable="False" strip="True">
+            #import re
+            #set nm=$bed_file_for_name.name
+            ${re.sub('\W',"_",nm)}
+        </param>
+        <param type="data" name="bed_file" url="${niceName}.bed" />
+        <param type="template" name="bioviz" strip="True" >
+            http://bioviz.org/galaxy.html?version=${bed_file.dbkey}&amp;loadresidues=false&amp;feature_url_0=${bed_file.url}&amp;sym_name_0=${niceName}&amp;sym_method_0=${bed_file.url}&amp;query_url=${bed_file.url}&amp;server_url=galaxy
+        </param>
+    </link>
+</display>

lib/galaxy/datatypes/display_applications/configs/igb/bedgraph.xml

@@ -0,0 +1,15 @@
+<display id="igb_bedgraph" version="1.0.0" name="display in IGB">
+    <link id="View" name="View">
+        <url>$bioviz</url>
+        <param type="data" name="bedgraph_file_for_name" viewable="False"/>
+        <param type="template" name="niceName" viewable="False" strip="True">
+            #import re
+            #set nm=$bedgraph_file_for_name.name
+            ${re.sub('\W',"_",nm)}
+        </param>
+        <param type="data" name="bedgraph_file" url="${niceName}.bed.bedgraph" />
+        <param type="template" name="bioviz" strip="True" >
+            http://bioviz.org/galaxy.html?version=${bedgraph_file.dbkey}&amp;loadresidues=false&amp;feature_url_0=${bedgraph_file.url}&amp;sym_name_0=${niceName}&amp;sym_method_0=${bedgraph_file.url}&amp;query_url=${bedgraph_file.url}&amp;server_url=galaxy
+        </param>
+    </link>
+</display>

lib/galaxy/datatypes/display_applications/configs/igb/bigwig.xml

@@ -0,0 +1,20 @@
+<display id="igb_bigwig" version="1.0.0" name="display in IGB">
+    <link id="View" name="View">
+        <url>http://bioviz.org/galaxy.html?version=${bigwig_file.dbkey}&amp;loadresidues=false&amp;feature_url_0=${bigwig_file.url}&amp;sym_name_0=${niceName}&amp;sym_method_0=${bigwig_file.url}&amp;query_url=${bigwig_file.url}&amp;server_url=galaxy</url>
+        <param type="data" name="bigwig_file_for_name" viewable="False"/>
+        <param type="template" name="niceName" viewable="False" strip="True">
+            #import re
+            #set nm=$bigwig_file_for_name.name
+            ${re.sub('\W',"_",nm)}
+        </param>
+        <param type="data" name="bigwig_file" url="${niceName}.bigwig" />
+        <!--<param type="template" name="position" strip="True" >
+            #set chrom, start, end = $bigwig_file.datatype.get_estimated_display_viewport( $bigwig_file )
+            #if $chrom is not None:
+                seqid=${chrom}&amp;start=${start}&amp;end=${int(end) + 1}
+            #else:
+                seqid=&amp;start=&amp;end=
+            #end if
+        </param>-->
+    </link>
+</display>

lib/galaxy/datatypes/display_applications/configs/igb/gtf.xml

@@ -0,0 +1,15 @@
+<display id="igb_gtf" version="1.0.0" name="display in IGB">
+    <link id="View" name="View">
+        <url>$bioviz</url>
+        <param type="data" name="gtf_file_for_name" viewable="False"/>
+        <param type="template" name="niceName" viewable="False" strip="True">
+            #import re
+            #set nm=$gtf_file_for_name.name
+            ${re.sub('\W',"_",nm)}
+        </param>
+        <param type="data" name="gtf_file" url="${niceName}.gtf" />
+        <param type="template" name="bioviz" strip="True" >
+             http://bioviz.org/galaxy.html?version=${gtf_file.dbkey}&amp;loadresidues=false&amp;feature_url_0=${gtf_file.url}&amp;sym_name_0=${niceName}&amp;sym_method_0=${gtf_file.url}&amp;query_url=${gtf_file.url}&amp;server_url=galaxy
+        </param>
+    </link>
+</display>

lib/galaxy/datatypes/display_applications/configs/igb/wig.xml

@@ -0,0 +1,20 @@
+<display id="igb_wig" version="1.0.0" name="display in IGB">
+    <link id="View" name="View">
+        <url>http://bioviz.org/galaxy.html?version=${wig_file.dbkey}&amp;${position}&amp;loadresidues=false&amp;feature_url_0=${wig_file.url}&amp;sym_name_0=${niceName}&amp;sym_method_0=${wig_file.url}&amp;query_url=${wig_file.url}&amp;server_url=galaxy</url>
+        <param type="data" name="wig_file_for_name" viewable="False"/>
+        <param type="template" name="niceName" viewable="False" strip="True">
+            #import re
+            #set nm=$wig_file_for_name.name
+            ${re.sub('\W',"_",nm)}
+        </param>
+        <param type="data" name="wig_file" url="${niceName}.wig" />
+        <param type="template" name="position" strip="True" >
+            #set chrom, start, end = $wig_file.datatype.get_estimated_display_viewport( $wig_file )
+            #if $chrom is not None:
+                seqid=${chrom}&amp;start=${start}&amp;end=${int(end) + 1}
+            #else:
+                seqid=&amp;start=&amp;end=
+            #end if
+        </param>
+    </link>
+</display>

lib/galaxy/datatypes/display_applications/configs/igv/bam.xml

@@ -0,0 +1,101 @@
+<?xml version="1.0"?>
+<display id="igv_bam" version="1.0.0" name="display with IGV">
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="igv" skip_startswith="#" id="0" name="1">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="site_name" value="1"/>
+        <dynamic_param name="site_link" value="2"/>
+        <dynamic_param name="site_dbkeys" value="3" split="True" separator="," />
+        <dynamic_param name="site_organisms" value="4" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_dbkeys to dataset dbkey -->
+        <filter>${$site_id.startswith( 'local_' ) or $dataset.dbkey in $site_dbkeys}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${redirect_url}</url>
+        <param type="data" name="bam_file" url="galaxy_${DATASET_HASH}.bam" />
+        <param type="data" name="bai_file" url="galaxy_${DATASET_HASH}.bam.bai" metadata="bam_index" />
+        <param type="template" name="site_organism" strip="True" >
+            #if ($dataset.dbkey in $site_dbkeys)
+                $site_organisms[ $site_dbkeys.index( $bam_file.dbkey ) ]
+            #else:
+                $bam_file.dbkey
+            #end if
+        </param>
+        <param type="template" name="site_organism_query_param" strip="True">
+            #if str($site_organism) not in ['','?']:
+                &amp;genome=${qp($site_organism)}
+            #end if
+        </param>
+        <param type="template" name="jnlp" url="galaxy_${DATASET_HASH}.jnlp" viewable="True" mimetype="application/x-java-jnlp-file"><![CDATA[
+<?xml version="1.0" encoding="utf-8"?>
+<jnlp
+  spec="1.0+"
+  codebase="${site_link}">
+  <information>
+    <title>IGV 1.5</title>
+    <vendor>The Broad Institute</vendor>
+    <homepage href="http://www.broadinstitute.org/igv"/>
+    <description>IGV Software</description>
+    <description kind="short">IGV</description>
+  </information>
+  <security>
+      <all-permissions/>
+  </security>
+  <resources>
+<j2se version="1.5+" initial-heap-size="256m" max-heap-size="1100m"/>
+    <jar href="igv.jar" download="eager" main="true"/>
+    <jar href="batik-codec.jar" download="eager"/>
+    <property name="apple.laf.useScreenMenuBar" value="true"/>
+    <property name="com.apple.mrj.application.growbox.intrudes" value="false"/>
+    <property name="com.apple.mrj.application.live-resize" value="true"/>
+    <property name="com.apple.macos.smallTabs" value="true"/>
+  </resources>
+    <resources os="Mac" arch="i386">
+        <property name="apple.awt.graphics.UseQuartz" value="false"/>
+        <nativelib href="hdfnative-macintel.jar"/>
+    </resources>
+    <resources os="Mac" arch="ppc">
+        <property name="apple.awt.graphics.UseQuartz" value="false"/>
+        <nativelib href="hdfnative-macppc.jar"/>
+    </resources>
+    <resources os="Mac" arch="PowerPC">
+        <property name="apple.awt.graphics.UseQuartz" value="false"/>
+        <nativelib href="hdfnative-macppc.jar"/>
+    </resources>
+    <resources os="Windows">
+        <property name="sun.java2d.noddraw" value="true"/>
+        <nativelib href="hdfnative-win.jar"/>
+    </resources>
+    <resources os="Linux">
+        <nativelib href="hdfnative-linux64.jar"/>
+    </resources>
+  <application-desc main-class="org.broad.igv.ui.IGVMainFrame">
+     #if str($site_organism) not in ['?','']:
+     <argument>-g</argument>
+     <argument>${site_organism}</argument>
+     #end if
+     <argument>${bam_file.url}</argument>
+  </application-desc>
+</jnlp>
+]]>
+        </param>
+        <param type="template" name="redirect_url" strip="True" >
+            #if $site_id.startswith( 'local_' )
+                ${site_link}?file=${bam_file.qp}${site_organism_query_param}&amp;merge=true&amp;name=${qp( ( $bam_file.name or $DATASET_HASH ).replace( ',', ';' ) )}
+            #elif $site_id.startswith( 'web_link_' ):
+                ${site_link}?sessionURL=${bam_file.qp}${site_organism_query_param}&amp;merge=true&amp;name=${qp( ( $bam_file.name or $DATASET_HASH ).replace( ',', ';' ) )}
+            #else:
+                ${jnlp.url}
+            #end if
+        </param>
+    </dynamic_links>
+    <dynamic_links from_data_table="igv_broad_genomes" skip_startswith="#" id="value" name="name">
+        <!-- Our input data table is one line per dbkey -->
+        <filter>${ $dataset.dbkey == $value }</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>http://www.broadinstitute.org/igv/projects/current/igv.php?sessionURL=${bam_file.qp}&amp;genome=${qp($bam_file.dbkey)}&amp;merge=true&amp;name=${qp( ( $bam_file.name or $DATASET_HASH ).replace( ',', ';' ) )}</url>
+        <param type="data" name="bam_file" url="galaxy_${DATASET_HASH}.bam" />
+        <param type="data" name="bai_file" url="galaxy_${DATASET_HASH}.bam.bai" metadata="bam_index" />
+    </dynamic_links>
+</display>
+<!-- Dan Blankenberg -->

lib/galaxy/datatypes/display_applications/configs/igv/bigwig.xml

@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<display id="igv_bigwig" version="1.0.0" name="display with IGV">
+    
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="igv" skip_startswith="#" id="0" name="1">
+        
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="site_name" value="1"/>
+        <dynamic_param name="site_link" value="2"/>
+        <dynamic_param name="site_dbkeys" value="3" split="True" separator=","/>
+        <dynamic_param name="site_organisms" value="4" split="True" separator=","/>
+        
+        <!-- Filter out some of the links based upon matching site_dbkeys to dataset dbkey -->
+<!--        <filter>${dataset.dbkey in $site_dbkeys}</filter> -->
+        
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${redirect_url}</url>
+        
+        <param type="data" name="bigwig_file" url="galaxy_${DATASET_HASH}.bw" format="bigwig"/>
+
+        <param type="template" name="jnlp" url="galaxy_${DATASET_HASH}.jnlp" viewable="True" mimetype="application/x-java-jnlp-file">&lt;?xml version="1.0" encoding="utf-8"?&gt;
+&lt;jnlp
+  spec="1.0+"
+  codebase="${site_link}"&gt;
+  &lt;information&gt;
+    &lt;title&gt;IGV 1.5&lt;/title&gt;
+    &lt;vendor&gt;The Broad Institute&lt;/vendor&gt;
+    &lt;homepage href="http://www.broadinstitute.org/igv"/&gt;
+    &lt;description&gt;IGV Software&lt;/description&gt;
+    &lt;description kind="short"&gt;IGV&lt;/description&gt;
+  &lt;/information&gt;
+  &lt;security&gt;
+      &lt;all-permissions/&gt;
+  &lt;/security&gt;
+  &lt;resources&gt;
+  
+&lt;j2se version="1.5+" initial-heap-size="256m" max-heap-size="1100m"/&gt;    
+	&lt;jar href="igv.jar" download="eager" main="true"/&gt;
+    &lt;jar href="batik-codec.jar" download="eager"/&gt; 
+    &lt;property name="apple.laf.useScreenMenuBar" value="true"/&gt;
+    &lt;property name="com.apple.mrj.application.growbox.intrudes" value="false"/&gt;
+    &lt;property name="com.apple.mrj.application.live-resize" value="true"/&gt;
+    &lt;property name="com.apple.macos.smallTabs" value="true"/&gt;
+  &lt;/resources&gt;
+	
+    &lt;resources os="Mac" arch="i386"&gt;
+        &lt;property name="apple.awt.graphics.UseQuartz" value="false"/&gt;
+        &lt;nativelib href="hdfnative-macintel.jar"/&gt; 
+    &lt;/resources&gt;
+
+    &lt;resources os="Mac" arch="ppc"&gt;
+        &lt;property name="apple.awt.graphics.UseQuartz" value="false"/&gt;
+        &lt;nativelib href="hdfnative-macppc.jar"/&gt; 
+    &lt;/resources&gt;
+    
+    &lt;resources os="Mac" arch="PowerPC"&gt;
+        &lt;property name="apple.awt.graphics.UseQuartz" value="false"/&gt;
+        &lt;nativelib href="hdfnative-macppc.jar"/&gt; 
+    &lt;/resources&gt;
+
+    &lt;resources os="Windows"&gt;
+        &lt;property name="sun.java2d.noddraw" value="true"/&gt;
+        &lt;nativelib href="hdfnative-win.jar"/&gt; 
+    &lt;/resources&gt;
+
+    &lt;resources os="Linux"&gt;
+        &lt;nativelib href="hdfnative-linux64.jar"/&gt; 
+    &lt;/resources&gt;
+
+  &lt;application-desc main-class="org.broad.igv.ui.IGVMainFrame"&gt;
+     &lt;argument&gt;${bigwig_file.url}&lt;/argument&gt;
+  &lt;/application-desc&gt;
+&lt;/jnlp&gt;
+		</param>
+        <param type="template" name="redirect_url" strip="True">
+            #if $site_id.startswith( 'local_' )
+                ${site_link}?file=${bigwig_file.qp}&amp;merge=true&amp;name=${qp( $bigwig_file.name )}
+            #elif $site_id.startswith( 'web_link_' ):
+                ${site_link}?sessionURL=${bigwig_file.qp}&amp;merge=true&amp;name=${qp( $bigwig_file.name )}
+            #else:
+                ${jnlp.url}
+            #end if
+        </param>
+    </dynamic_links>
+    
+    
+</display>
+<!-- Ann Black-Ziegelbein based on Dan Blankenberg -->

lib/galaxy/datatypes/display_applications/configs/igv/genbank.xml

@@ -0,0 +1,99 @@
+<?xml version="1.0"?>
+<display id="igv_genbank" version="1.0.0" name="display with IGV">
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="igv" skip_startswith="#" id="0" name="1">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="site_name" value="1"/>
+        <dynamic_param name="site_link" value="2"/>
+        <dynamic_param name="site_dbkeys" value="3" split="True" separator="," />
+        <dynamic_param name="site_organisms" value="4" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_dbkeys to dataset dbkey -->
+        <filter>${$site_id.startswith( 'local_' ) or $dataset.dbkey in $site_dbkeys}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${redirect_url}</url>
+        <param type="data" name="genbank_file" url="galaxy_${DATASET_HASH}.gb" format="genbank" />
+        <param type="template" name="site_organism" strip="True" >
+            #if ($dataset.dbkey in $site_dbkeys)
+                $site_organisms[ $site_dbkeys.index( $genbank_file.dbkey ) ]
+            #else:
+                $genbank_file.dbkey
+            #end if
+        </param>
+        <param type="template" name="site_organism_query_param" strip="True">
+            #if str($site_organism) not in ['','?']:
+                &amp;genome=${qp($site_organism)}
+            #end if
+        </param>
+        <param type="template" name="jnlp" url="galaxy_${DATASET_HASH}.jnlp" viewable="True" mimetype="application/x-java-jnlp-file"><![CDATA[
+<?xml version="1.0" encoding="utf-8"?>
+<jnlp
+  spec="1.0+"
+  codebase="${site_link}">
+  <information>
+    <title>IGV 1.5</title>
+    <vendor>The Broad Institute</vendor>
+    <homepage href="http://www.broadinstitute.org/igv"/>
+    <description>IGV Software</description>
+    <description kind="short">IGV</description>
+  </information>
+  <security>
+      <all-permissions/>
+  </security>
+  <resources>
+<j2se version="1.5+" initial-heap-size="256m" max-heap-size="1100m"/>
+    <jar href="igv.jar" download="eager" main="true"/>
+    <jar href="batik-codec.jar" download="eager"/>
+    <property name="apple.laf.useScreenMenuBar" value="true"/>
+    <property name="com.apple.mrj.application.growbox.intrudes" value="false"/>
+    <property name="com.apple.mrj.application.live-resize" value="true"/>
+    <property name="com.apple.macos.smallTabs" value="true"/>
+  </resources>
+    <resources os="Mac" arch="i386">
+        <property name="apple.awt.graphics.UseQuartz" value="false"/>
+        <nativelib href="hdfnative-macintel.jar"/>
+    </resources>
+    <resources os="Mac" arch="ppc">
+        <property name="apple.awt.graphics.UseQuartz" value="false"/>
+        <nativelib href="hdfnative-macppc.jar"/>
+    </resources>
+    <resources os="Mac" arch="PowerPC">
+        <property name="apple.awt.graphics.UseQuartz" value="false"/>
+        <nativelib href="hdfnative-macppc.jar"/>
+    </resources>
+    <resources os="Windows">
+        <property name="sun.java2d.noddraw" value="true"/>
+        <nativelib href="hdfnative-win.jar"/>
+    </resources>
+    <resources os="Linux">
+        <nativelib href="hdfnative-linux64.jar"/>
+    </resources>
+  <application-desc main-class="org.broad.igv.ui.IGVMainFrame">
+     #if str($site_organism) not in ['?','']:
+     <argument>-g</argument>
+     <argument>${site_organism}</argument>
+     #end if
+     <argument>${genbank_file.url}</argument>
+  </application-desc>
+</jnlp>
+]]>
+        </param>
+        <param type="template" name="redirect_url" strip="True" >
+            #if $site_id.startswith( 'local_' )
+                ${site_link}?file=${genbank_file.qp}${site_organism_query_param}&amp;merge=true&amp;name=${qp( ( $genbank_file.name or $DATASET_HASH ).replace( ',', ';' ) )}
+            #elif $site_id.startswith( 'web_link_' ):
+                ${site_link}?sessionURL=${genbank_file.qp}${site_organism_query_param}&amp;merge=true&amp;name=${qp( ( $genbank_file.name or $DATASET_HASH ).replace( ',', ';' ) )}
+            #else:
+                ${jnlp.url}
+            #end if
+        </param>
+    </dynamic_links>
+    <dynamic_links from_data_table="igv_broad_genomes" skip_startswith="#" id="value" name="name">
+        <!-- Our input data table is one line per dbkey -->
+        <filter>${ $dataset.dbkey == $value }</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>http://www.broadinstitute.org/igv/projects/current/igv.php?sessionURL=${genbank_file.qp}&amp;genome=${qp($genbank_file.dbkey)}&amp;merge=true&amp;name=${qp( ( $genbank_file.name or $DATASET_HASH ).replace( ',', ';' ) )}</url>
+        <param type="data" name="genbank_file" url="galaxy_${DATASET_HASH}.gb" format="genbank" />
+    </dynamic_links>
+</display>
+<!-- Dan Blankenberg -->

lib/galaxy/datatypes/display_applications/configs/igv/genome_fasta.xml

@@ -0,0 +1,92 @@
+<?xml version="1.0"?>
+<display id="igv_fasta" version="1.0.0" name="display with IGV">
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="igv" skip_startswith="#" id="0" name="1">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="site_name" value="1"/>
+        <dynamic_param name="site_link" value="2"/>
+        <dynamic_param name="site_dbkeys" value="3" split="True" separator="," />
+        <dynamic_param name="site_organisms" value="4" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_dbkeys to dataset dbkey -->
+        <filter>${$site_id.startswith( 'local_' ) or $dataset.dbkey in $site_dbkeys}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${redirect_url}</url>
+        <param type="data" name="fasta_file" url="galaxy_${DATASET_HASH}.${dataset.ext}" />
+        <param type="template" name="site_organism" strip="True" >
+            #if ($dataset.dbkey in $site_dbkeys)
+                $site_organisms[ $site_dbkeys.index( $fasta_file.dbkey ) ]
+            #else:
+                $fasta_file.dbkey
+            #end if
+        </param>
+        <param type="template" name="jnlp" url="galaxy_${DATASET_HASH}.jnlp" viewable="True" mimetype="application/x-java-jnlp-file"><![CDATA[
+<?xml version="1.0" encoding="utf-8"?>
+<jnlp
+  spec="1.0+"
+  codebase="${site_link}">
+  <information>
+    <title>IGV 1.5</title>
+    <vendor>The Broad Institute</vendor>
+    <homepage href="http://www.broadinstitute.org/igv"/>
+    <description>IGV Software</description>
+    <description kind="short">IGV</description>
+  </information>
+  <security>
+      <all-permissions/>
+  </security>
+  <resources>
+<j2se version="1.5+" initial-heap-size="256m" max-heap-size="1100m"/>
+    <jar href="igv.jar" download="eager" main="true"/>
+    <jar href="batik-codec.jar" download="eager"/>
+    <property name="apple.laf.useScreenMenuBar" value="true"/>
+    <property name="com.apple.mrj.application.growbox.intrudes" value="false"/>
+    <property name="com.apple.mrj.application.live-resize" value="true"/>
+    <property name="com.apple.macos.smallTabs" value="true"/>
+  </resources>
+    <resources os="Mac" arch="i386">
+        <property name="apple.awt.graphics.UseQuartz" value="false"/>
+        <nativelib href="hdfnative-macintel.jar"/>
+    </resources>
+    <resources os="Mac" arch="ppc">
+        <property name="apple.awt.graphics.UseQuartz" value="false"/>
+        <nativelib href="hdfnative-macppc.jar"/>
+    </resources>
+    <resources os="Mac" arch="PowerPC">
+        <property name="apple.awt.graphics.UseQuartz" value="false"/>
+        <nativelib href="hdfnative-macppc.jar"/>
+    </resources>
+    <resources os="Windows">
+        <property name="sun.java2d.noddraw" value="true"/>
+        <nativelib href="hdfnative-win.jar"/>
+    </resources>
+    <resources os="Linux">
+        <nativelib href="hdfnative-linux64.jar"/>
+    </resources>
+  <application-desc main-class="org.broad.igv.ui.IGVMainFrame">
+     <argument>-g</argument>
+     <argument>${fasta_file.url}</argument>
+  </application-desc>
+</jnlp>
+]]>
+        </param>
+        <param type="data" name="fai_file" dataset="fasta_file" url="galaxy_${DATASET_HASH}.${fasta_file.ext}.fai" format="fai" />
+        <param type="template" name="redirect_url" strip="True" >
+            #if $site_id.startswith( 'local_' )
+                ${site_link}?genome=${fasta_file.qp}&amp;merge=true&amp;name=${qp( ( $fasta_file.name or $DATASET_HASH ).replace( ',', ';' ) )}
+            #elif $site_id.startswith( 'web_link_' ):
+${site_link}?sessionURL=${fasta_file.qp}&amp;genome=${fasta_file.qp}&amp;merge=true&amp;name=${qp( ( $fasta_file.name or $DATASET_HASH ).replace( ',', ';' ) )}
+            #else:
+                ${jnlp.url}
+            #end if
+        </param>
+    </dynamic_links>
+    <dynamic_links from_data_table="igv_broad_genomes" skip_startswith="#" id="value" name="name">
+        <!-- Our input data table is one line per dbkey -->
+        <filter>${ $dataset.dbkey == $value }</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>http://www.broadinstitute.org/igv/projects/current/igv.php?sessionURL=${fasta_file.qp}&amp;genome=${fasta_file.dbkey}&amp;merge=true&amp;name=${qp( ( $fasta_file.name or $DATASET_HASH ).replace( ',', ';' ) )}</url>
+        <param type="data" name="fasta_file" url="galaxy_${DATASET_HASH}.${dataset.ext}" />
+    </dynamic_links>
+</display>
+<!-- Delphine Lariviere - Dan Blankenberg -->

lib/galaxy/datatypes/display_applications/configs/igv/gff.xml

@@ -0,0 +1,99 @@
+<?xml version="1.0"?>
+<display id="igv_gff" version="1.0.0" name="display with IGV">
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="igv" skip_startswith="#" id="0" name="1">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="site_name" value="1"/>
+        <dynamic_param name="site_link" value="2"/>
+        <dynamic_param name="site_dbkeys" value="3" split="True" separator="," />
+        <dynamic_param name="site_organisms" value="4" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_dbkeys to dataset dbkey -->
+        <filter>${$site_id.startswith( 'local_' ) or $dataset.dbkey in $site_dbkeys}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${redirect_url}</url>
+        <param type="data" name="gff_file" url="galaxy_${DATASET_HASH}.${dataset.ext}" />
+        <param type="template" name="site_organism" strip="True" >
+            #if ($dataset.dbkey in $site_dbkeys)
+                $site_organisms[ $site_dbkeys.index( $gff_file.dbkey ) ]
+            #else:
+                $gff_file.dbkey
+            #end if
+        </param>
+        <param type="template" name="site_organism_query_param" strip="True">
+            #if str($site_organism) not in ['','?']:
+                &amp;genome=${qp($site_organism)}
+            #end if
+        </param>
+        <param type="template" name="jnlp" url="galaxy_${DATASET_HASH}.jnlp" viewable="True" mimetype="application/x-java-jnlp-file"><![CDATA[
+<?xml version="1.0" encoding="utf-8"?>
+<jnlp
+  spec="1.0+"
+  codebase="${site_link}">
+  <information>
+    <title>IGV 1.5</title>
+    <vendor>The Broad Institute</vendor>
+    <homepage href="http://www.broadinstitute.org/igv"/>
+    <description>IGV Software</description>
+    <description kind="short">IGV</description>
+  </information>
+  <security>
+      <all-permissions/>
+  </security>
+  <resources>
+<j2se version="1.5+" initial-heap-size="256m" max-heap-size="1100m"/>
+    <jar href="igv.jar" download="eager" main="true"/>
+    <jar href="batik-codec.jar" download="eager"/>
+    <property name="apple.laf.useScreenMenuBar" value="true"/>
+    <property name="com.apple.mrj.application.growbox.intrudes" value="false"/>
+    <property name="com.apple.mrj.application.live-resize" value="true"/>
+    <property name="com.apple.macos.smallTabs" value="true"/>
+  </resources>
+    <resources os="Mac" arch="i386">
+        <property name="apple.awt.graphics.UseQuartz" value="false"/>
+        <nativelib href="hdfnative-macintel.jar"/>
+    </resources>
+    <resources os="Mac" arch="ppc">
+        <property name="apple.awt.graphics.UseQuartz" value="false"/>
+        <nativelib href="hdfnative-macppc.jar"/>
+    </resources>
+    <resources os="Mac" arch="PowerPC">
+        <property name="apple.awt.graphics.UseQuartz" value="false"/>
+        <nativelib href="hdfnative-macppc.jar"/>
+    </resources>
+    <resources os="Windows">
+        <property name="sun.java2d.noddraw" value="true"/>
+        <nativelib href="hdfnative-win.jar"/>
+    </resources>
+    <resources os="Linux">
+        <nativelib href="hdfnative-linux64.jar"/>
+    </resources>
+  <application-desc main-class="org.broad.igv.ui.IGVMainFrame">
+     #if str($site_organism) not in ['?','']:
+     <argument>-g</argument>
+     <argument>${site_organism}</argument>
+     #end if
+     <argument>${gff_file.url}</argument>
+  </application-desc>
+</jnlp>
+]]>
+        </param>
+        <param type="template" name="redirect_url" strip="True" >
+            #if $site_id.startswith( 'local_' )
+                ${site_link}?file=${gff_file.qp}${site_organism_query_param}&amp;merge=true&amp;name=${qp( ( $gff_file.name or $DATASET_HASH ).replace( ',', ';' ) )}
+            #elif $site_id.startswith( 'web_link_' ):
+                ${site_link}?sessionURL=${gff_file.qp}${site_organism_query_param}&amp;merge=true&amp;name=${qp( ( $gff_file.name or $DATASET_HASH ).replace( ',', ';' ) )}
+            #else:
+                ${jnlp.url}
+            #end if
+        </param>
+    </dynamic_links>
+    <dynamic_links from_data_table="igv_broad_genomes" skip_startswith="#" id="value" name="name">
+        <!-- Our input data table is one line per dbkey -->
+        <filter>${ $dataset.dbkey == $value }</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>http://www.broadinstitute.org/igv/projects/current/igv.php?sessionURL=${gff_file.qp}&amp;genome=${qp($gff_file.dbkey)}&amp;merge=true&amp;name=${qp( ( $gff_file.name or $DATASET_HASH ).replace( ',', ';' ) )}</url>
+        <param type="data" name="gff_file" url="galaxy_${DATASET_HASH}.${dataset.ext}" />
+    </dynamic_links>
+</display>
+<!-- Dan Blankenberg -->

lib/galaxy/datatypes/display_applications/configs/igv/interval_as_bed.xml

@@ -0,0 +1,99 @@
+<?xml version="1.0"?>
+<display id="igv_interval_as_bed" version="1.0.0" name="display with IGV">
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="igv" skip_startswith="#" id="0" name="1">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="site_name" value="1"/>
+        <dynamic_param name="site_link" value="2"/>
+        <dynamic_param name="site_dbkeys" value="3" split="True" separator="," />
+        <dynamic_param name="site_organisms" value="4" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_dbkeys to dataset dbkey -->
+        <filter>${$site_id.startswith( 'local_' ) or $dataset.dbkey in $site_dbkeys}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${redirect_url}</url>
+        <param type="data" name="bed_file" url="galaxy_${DATASET_HASH}.bed" format="bedstrict" />
+        <param type="template" name="site_organism" strip="True" >
+            #if ($dataset.dbkey in $site_dbkeys)
+                $site_organisms[ $site_dbkeys.index( $bed_file.dbkey ) ]
+            #else:
+                $bed_file.dbkey
+            #end if
+        </param>
+        <param type="template" name="site_organism_query_param" strip="True">
+            #if str($site_organism) not in ['','?']:
+                &amp;genome=${qp($site_organism)}
+            #end if
+        </param>
+        <param type="template" name="jnlp" url="galaxy_${DATASET_HASH}.jnlp" viewable="True" mimetype="application/x-java-jnlp-file"><![CDATA[
+<?xml version="1.0" encoding="utf-8"?>
+<jnlp
+  spec="1.0+"
+  codebase="${site_link}">
+  <information>
+    <title>IGV 1.5</title>
+    <vendor>The Broad Institute</vendor>
+    <homepage href="http://www.broadinstitute.org/igv"/>
+    <description>IGV Software</description>
+    <description kind="short">IGV</description>
+  </information>
+  <security>
+      <all-permissions/>
+  </security>
+  <resources>
+<j2se version="1.5+" initial-heap-size="256m" max-heap-size="1100m"/>
+    <jar href="igv.jar" download="eager" main="true"/>
+    <jar href="batik-codec.jar" download="eager"/>
+    <property name="apple.laf.useScreenMenuBar" value="true"/>
+    <property name="com.apple.mrj.application.growbox.intrudes" value="false"/>
+    <property name="com.apple.mrj.application.live-resize" value="true"/>
+    <property name="com.apple.macos.smallTabs" value="true"/>
+  </resources>
+    <resources os="Mac" arch="i386">
+        <property name="apple.awt.graphics.UseQuartz" value="false"/>
+        <nativelib href="hdfnative-macintel.jar"/>
+    </resources>
+    <resources os="Mac" arch="ppc">
+        <property name="apple.awt.graphics.UseQuartz" value="false"/>
+        <nativelib href="hdfnative-macppc.jar"/>
+    </resources>
+    <resources os="Mac" arch="PowerPC">
+        <property name="apple.awt.graphics.UseQuartz" value="false"/>
+        <nativelib href="hdfnative-macppc.jar"/>
+    </resources>
+    <resources os="Windows">
+        <property name="sun.java2d.noddraw" value="true"/>
+        <nativelib href="hdfnative-win.jar"/>
+    </resources>
+    <resources os="Linux">
+        <nativelib href="hdfnative-linux64.jar"/>
+    </resources>
+  <application-desc main-class="org.broad.igv.ui.IGVMainFrame">
+     #if str($site_organism) not in ['?','']:
+     <argument>-g</argument>
+     <argument>${site_organism}</argument>
+     #end if
+     <argument>${bed_file.url}</argument>
+  </application-desc>
+</jnlp>
+]]>
+        </param>
+        <param type="template" name="redirect_url" strip="True" >
+            #if $site_id.startswith( 'local_' )
+                ${site_link}?file=${bed_file.qp}${site_organism_query_param}&amp;merge=true&amp;name=${qp( ( $bed_file.name or $DATASET_HASH ).replace( ',', ';' ) )}
+            #elif $site_id.startswith( 'web_link_' ):
+                ${site_link}?sessionURL=${bed_file.qp}${site_organism_query_param}&amp;merge=true&amp;name=${qp( ( $bed_file.name or $DATASET_HASH ).replace( ',', ';' ) )}
+            #else:
+                ${jnlp.url}
+            #end if
+        </param>
+    </dynamic_links>
+    <dynamic_links from_data_table="igv_broad_genomes" skip_startswith="#" id="value" name="name">
+        <!-- Our input data table is one line per dbkey -->
+        <filter>${ $dataset.dbkey == $value }</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>http://www.broadinstitute.org/igv/projects/current/igv.php?sessionURL=${bed_file.qp}&amp;genome=${qp($bed_file.dbkey)}&amp;merge=true&amp;name=${qp( ( $bed_file.name or $DATASET_HASH ).replace( ',', ';' ) )}</url>
+        <param type="data" name="bed_file" url="galaxy_${DATASET_HASH}.bed" format="bedstrict" />
+    </dynamic_links>
+</display>
+<!-- Dan Blankenberg -->

lib/galaxy/datatypes/display_applications/configs/igv/vcf.xml

@@ -0,0 +1,101 @@
+<?xml version="1.0"?>
+<display id="igv_vcf" version="1.0.0" name="display with IGV">
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="igv" skip_startswith="#" id="0" name="1">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="site_name" value="1"/>
+        <dynamic_param name="site_link" value="2"/>
+        <dynamic_param name="site_dbkeys" value="3" split="True" separator="," />
+        <dynamic_param name="site_organisms" value="4" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_dbkeys to dataset dbkey -->
+        <filter>${$site_id.startswith( 'local_' ) or $dataset.dbkey in $site_dbkeys}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${redirect_url}</url>
+        <param type="data" name="bgzip_file" url="galaxy_${DATASET_HASH}.vcf.gz" format="vcf_bgzip" allow_cors="true" mimetype="application/octet-stream"/>
+        <param type="data" name="tabix_file" metadata="tabix_index" url="galaxy_${DATASET_HASH}.vcf.gz.tbi" allow_cors="true" mimetype="application/octet-stream"/>
+        <param type="template" name="site_organism" strip="True" >
+            #if ($dataset.dbkey in $site_dbkeys)
+                $site_organisms[ $site_dbkeys.index( $bgzip_file.dbkey ) ]
+            #else:
+                $bgzip_file.dbkey
+            #end if
+        </param>
+        <param type="template" name="site_organism_query_param" strip="True">
+            #if str($site_organism) not in ['','?']:
+                &amp;genome=${qp($site_organism)}
+            #end if
+        </param>
+        <param type="template" name="jnlp" url="galaxy_${DATASET_HASH}.jnlp" viewable="True" mimetype="application/x-java-jnlp-file"><![CDATA[
+<?xml version="1.0" encoding="utf-8"?>
+<jnlp
+  spec="1.0+"
+  codebase="${site_link}">
+  <information>
+    <title>IGV 1.5</title>
+    <vendor>The Broad Institute</vendor>
+    <homepage href="http://www.broadinstitute.org/igv"/>
+    <description>IGV Software</description>
+    <description kind="short">IGV</description>
+  </information>
+  <security>
+      <all-permissions/>
+  </security>
+  <resources>
+<j2se version="1.5+" initial-heap-size="256m" max-heap-size="1100m"/>
+    <jar href="igv.jar" download="eager" main="true"/>
+    <jar href="batik-codec.jar" download="eager"/>
+    <property name="apple.laf.useScreenMenuBar" value="true"/>
+    <property name="com.apple.mrj.application.growbox.intrudes" value="false"/>
+    <property name="com.apple.mrj.application.live-resize" value="true"/>
+    <property name="com.apple.macos.smallTabs" value="true"/>
+  </resources>
+    <resources os="Mac" arch="i386">
+        <property name="apple.awt.graphics.UseQuartz" value="false"/>
+        <nativelib href="hdfnative-macintel.jar"/>
+    </resources>
+    <resources os="Mac" arch="ppc">
+        <property name="apple.awt.graphics.UseQuartz" value="false"/>
+        <nativelib href="hdfnative-macppc.jar"/>
+    </resources>
+    <resources os="Mac" arch="PowerPC">
+        <property name="apple.awt.graphics.UseQuartz" value="false"/>
+        <nativelib href="hdfnative-macppc.jar"/>
+    </resources>
+    <resources os="Windows">
+        <property name="sun.java2d.noddraw" value="true"/>
+        <nativelib href="hdfnative-win.jar"/>
+    </resources>
+    <resources os="Linux">
+        <nativelib href="hdfnative-linux64.jar"/>
+    </resources>
+  <application-desc main-class="org.broad.igv.ui.IGVMainFrame">
+     #if str($site_organism) not in ['?','']:
+     <argument>-g</argument>
+     <argument>${site_organism}</argument>
+     #end if
+     <argument>${bgzip_file.url}</argument>
+  </application-desc>
+</jnlp>
+]]>
+        </param>
+        <param type="template" name="redirect_url" strip="True" >
+            #if $site_id.startswith( 'local_' )
+                ${site_link}?file=${bgzip_file.qp}${site_organism_query_param}&amp;merge=true&amp;name=${qp( ( $bgzip_file.name or $DATASET_HASH ).replace( ',', ';' ) )}
+            #elif $site_id.startswith( 'web_link_' ):
+                ${site_link}?sessionURL=${bgzip_file.qp}${site_organism_query_param}&amp;merge=true&amp;name=${qp( ( $bgzip_file.name or $DATASET_HASH ).replace( ',', ';' ) )}
+            #else:
+                ${jnlp.url}
+            #end if
+        </param>
+    </dynamic_links>
+    <dynamic_links from_data_table="igv_broad_genomes" skip_startswith="#" id="value" name="name">
+        <!-- Our input data table is one line per dbkey -->
+        <filter>${ $dataset.dbkey == $value }</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>http://www.broadinstitute.org/igv/projects/current/igv.php?sessionURL=${bgzip_file.qp}&amp;genome=${qp($bgzip_file.dbkey)}&amp;merge=true&amp;name=${qp( ( $bgzip_file.name or $DATASET_HASH ).replace( ',', ';' ) )}</url>
+        <param type="data" name="bgzip_file" url="galaxy_${DATASET_HASH}.vcf.gz" format="vcf_bgzip" mimetype="application/octet-stream"/>
+        <param type="data" name="tabix_file" metadata="tabix_index" url="galaxy_${DATASET_HASH}.vcf.gz.tbi" format="vcf_bgzip" mimetype="application/octet-stream"/>
+    </dynamic_links>
+</display>
+<!-- Dan Blankenberg -->

lib/galaxy/datatypes/display_applications/configs/image/avivator.xml

@@ -0,0 +1,7 @@
+<display id="avivator" version="1.0.0" name="display at">
+    <link id="main" name="Avivator">
+        <url>http://avivator.gehlenborglab.org/?image_url=${qp($tiff_file.url)}</url>
+        <param type="data" name="tiff_file" url="galaxy_${DATASET_HASH}.ome.tiff" allow_cors="true"/>
+        <param type="data" name="offsets_file" url="galaxy_${DATASET_HASH}.offsets.json" metadata="offsets" allow_cors="true"/>
+    </link>
+</display>

lib/galaxy/datatypes/display_applications/configs/intermine/intermine_simple.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0"?>
+<display id="intermine_simple" version="1.0.0" name="view intermine at">
+    <dynamic_links from_data_table="intermine_simple_display" skip_startswith="#" id="value" name="name">
+        <url>${ url % { 'intermine_file_url_qp': $intermine_file.qp } }</url>
+        <param type="data" name="intermine_file" url="galaxy_${DATASET_HASH}.intermine" allow_cors="true" />
+    </dynamic_links>
+</display>

lib/galaxy/datatypes/display_applications/configs/iobio/bam.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<display id="iobio_bam" version="1.0.0" name="display at bam.iobio">
+    <dynamic_links from_data_table="bam_iobio" skip_startswith="#" id="value" name="name">
+        <url>${url}?bam=${bam_file.qp}</url>
+        <param type="data" name="bam_file" url="galaxy_${DATASET_HASH}.bam" />
+        <param type="data" name="bai_file" url="galaxy_${DATASET_HASH}.bam.bai" metadata="bam_index" />
+    </dynamic_links>
+</display>

lib/galaxy/datatypes/display_applications/configs/iobio/vcf.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<display id="iobio_vcf" version="1.0.0" name="display at vcf.iobio">
+    <dynamic_links from_data_table="vcf_iobio" skip_startswith="#" id="value" name="name">
+        <url>${url}?vcf=${bgzip_file.qp}</url>
+        <param type="data" name="bgzip_file" url="galaxy_${DATASET_HASH}.vcf.gz" format="vcf_bgzip" />
+        <param type="data" name="tabix_file" dataset="bgzip_file" url="galaxy_${DATASET_HASH}.vcf.gz.tbi" format="tabix" />
+    </dynamic_links>
+</display>

lib/galaxy/datatypes/display_applications/configs/rviewer/bed.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+<display id="rviewer_interval" version="1.0.0" name="display at RViewer">
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="rviewer" skip_startswith="#" id="0" name="1">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="site_name" value="1"/>
+        <dynamic_param name="site_link" value="2"/>
+        <dynamic_param name="site_dbkeys" value="3" split="True" separator="," />
+        <dynamic_param name="site_genome_versions" value="4" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_dbkeys to dataset dbkey -->
+        <filter>${dataset.dbkey in $site_dbkeys}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${site_link}/application/service/datasets/add?datasetName=${qp($bed_file.name)}&amp;genomeVersion=${qp( $site_genome_versions[ $site_dbkeys.index( $bed_file.dbkey ) ] )}&amp;dataFormat=bed&amp;dataUrl=${bed_file.qp}</url>
+        <param type="data" name="bed_file" format="bed" url="galaxy.bed" />
+    </dynamic_links>
+</display>

lib/galaxy/datatypes/display_applications/configs/rviewer/vcf.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+<display id="rviewer_vcf" version="1.0.0" name="display at RViewer">
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="rviewer" skip_startswith="#" id="0" name="1">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="site_name" value="1"/>
+        <dynamic_param name="site_link" value="2"/>
+        <dynamic_param name="site_dbkeys" value="3" split="True" separator="," />
+        <dynamic_param name="site_genome_versions" value="4" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_dbkeys to dataset dbkey -->
+        <filter>${dataset.dbkey in $site_dbkeys}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${site_link}/application/service/datasets/add?datasetName=${qp($vcf_file.name)}&amp;genomeVersion=${qp( $site_genome_versions[ $site_dbkeys.index( $vcf_file.dbkey ) ] )}&amp;dataFormat=vcf&amp;dataUrl=${vcf_file.qp}</url>
+        <param type="data" name="vcf_file" url="galaxy.vcf" />
+    </dynamic_links>
+</display>

lib/galaxy/datatypes/display_applications/configs/ucsc/bam.xml

@@ -0,0 +1,17 @@
+<display id="ucsc_bam" version="1.0.0" name="display at UCSC">
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="ucsc" skip_startswith="#" id="0" name="0">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="ucsc_link" value="1"/>
+        <dynamic_param name="builds" value="2" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_id to a Galaxy application configuration parameter and by dataset dbkey -->
+        <filter>${site_id in $APP.datatypes_registry.get_display_sites('ucsc')}</filter>
+        <filter>${dataset.dbkey in $builds}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${ucsc_link}db=${qp($bam_file.dbkey)}&amp;hgt.customText=${qp($track.url)}</url>
+        <param type="data" name="bam_file" url="galaxy_${DATASET_HASH}.bam" />
+        <param type="data" name="bai_file" url="galaxy_${DATASET_HASH}.bam.bai" metadata="bam_index" /><!-- UCSC expects index file to exist as bam_file_name.bai -->
+        <param type="template" name="track" viewable="True">track type="bam" name="${bam_file.name.replace( '\\', '\\\\' ).replace( '"', '\\"' )}" bigDataUrl="${bam_file.url}" db="${bam_file.dbkey}" pairEndsByName="."</param>
+    </dynamic_links>
+</display>

lib/galaxy/datatypes/display_applications/configs/ucsc/bigbed.xml

@@ -0,0 +1,16 @@
+<display id="ucsc_bigbed" version="1.0.0" name="display at UCSC">
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="ucsc" skip_startswith="#" id="0" name="0">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="ucsc_link" value="1"/>
+        <dynamic_param name="builds" value="2" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_id to a Galaxy application configuration parameter and by dataset dbkey -->
+        <filter>${site_id in $APP.datatypes_registry.get_display_sites('ucsc')}</filter>
+        <filter>${dataset.dbkey in $builds}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${ucsc_link}db=${qp($bigbed_file.dbkey)}&amp;hgt.customText=${qp($track.url)}</url>
+        <param type="data" name="bigbed_file" url="galaxy_${DATASET_HASH}.bigbed" />
+        <param type="template" name="track" viewable="True">track type="bigBed" name="${bigbed_file.name.replace( '\\', '\\\\' ).replace( '"', '\\"' )}" bigDataUrl="${bigbed_file.url}" db="${bigbed_file.dbkey}"</param>
+    </dynamic_links>
+</display>

lib/galaxy/datatypes/display_applications/configs/ucsc/bigwig.xml

@@ -0,0 +1,16 @@
+<display id="ucsc_bigwig" version="1.0.0" name="display at UCSC">
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="ucsc" skip_startswith="#" id="0" name="0">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="ucsc_link" value="1"/>
+        <dynamic_param name="builds" value="2" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_id to a Galaxy application configuration parameter and by dataset dbkey -->
+        <filter>${site_id in $APP.datatypes_registry.get_display_sites('ucsc')}</filter>
+        <filter>${dataset.dbkey in $builds}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${ucsc_link}db=${qp($bigwig_file.dbkey)}&amp;hgt.customText=${qp($track.url)}</url>
+        <param type="data" name="bigwig_file" url="galaxy_${DATASET_HASH}.bigwig" />
+        <param type="template" name="track" viewable="True">track type="bigWig" name="${bigwig_file.name.replace( '\\', '\\\\' ).replace( '"', '\\"' )}" bigDataUrl="${bigwig_file.url}" db="${bigwig_file.dbkey}"</param>
+    </dynamic_links>
+</display>

lib/galaxy/datatypes/display_applications/configs/ucsc/interval_as_bed.xml

@@ -0,0 +1,23 @@
+<display id="ucsc_interval_as_bed" version="1.0.0" name="display at UCSC">
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="ucsc" skip_startswith="#" id="0" name="0">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="ucsc_link" value="1"/>
+        <dynamic_param name="builds" value="2" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_id to a Galaxy application configuration parameter and by dataset dbkey -->
+        <filter>${site_id in $APP.datatypes_registry.get_display_sites('ucsc')}</filter>
+        <filter>${dataset.dbkey in $builds}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${ucsc_link}db=${qp($bed_file.dbkey)}&amp;position=${position.qp}&amp;hgt.customText=${bed_file.qp}</url>
+        <param type="data" name="bed_file" url="galaxy_${DATASET_HASH}.bed" format="bedstrict"/> <!-- Galaxy allows BED files to contain non-standard fields beyond the first 3 columns, UCSC does not: force use of converter which will make strict BED6+ file -->
+        <param type="template" name="position" strip="True" >
+#set chrom, start, end = $bed_file.datatype.get_estimated_display_viewport( $bed_file )
+#if $chrom is not None:
+${chrom}:${start}-${int( end ) + 1}
+#else:
+:-
+#end if
+        </param>
+    </dynamic_links>
+</display>

lib/galaxy/datatypes/display_applications/configs/ucsc/maf_customtrack.xml

@@ -0,0 +1,15 @@
+<display id="ucsc_maf_customtrack" version="1.0.0" name="display at UCSC">
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="ucsc" skip_startswith="#" id="0" name="0">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="ucsc_link" value="1"/>
+        <dynamic_param name="builds" value="2" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_id to a Galaxy application configuration parameter and by dataset dbkey -->
+        <filter>${site_id in $APP.datatypes_registry.get_display_sites('ucsc')}</filter>
+        <filter>${dataset.dbkey in $builds}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${ucsc_link}db=${qp($maf_customtrack_file.dbkey)}&amp;position=${maf_customtrack_file.metadata.vp_chromosome}:${maf_customtrack_file.metadata.vp_start}-${maf_customtrack_file.metadata.vp_end}&amp;hgt.customText=${maf_customtrack_file.qp}</url>
+        <param type="data" name="maf_customtrack_file" url="galaxy_${DATASET_HASH}.maf" format="mafcustomtrack"/>
+    </dynamic_links>
+</display>

lib/galaxy/datatypes/display_applications/configs/ucsc/trackhub.xml

@@ -0,0 +1,6 @@
+<display id="ucsc_trackhub" version="1.0.0" name="display at Track Hub UCSC">
+    <link id="main" name="main">
+        <url>https://genome.ucsc.edu/cgi-bin/hgHubConnect?hubUrl=${qp($hub_file.url + '/myHub/hub.txt')}&amp;hgHub_do_firstDb=on&amp;hgHub_do_redirect=on&amp;hgHubConnect.remakeTrackHub=on</url>
+        <param type="data" name="hub_file" url="galaxy_${DATASET_HASH}" allow_extra_files_access="True" />
+    </link>
+</display>

lib/galaxy/datatypes/display_applications/configs/ucsc/vcf.xml

@@ -0,0 +1,17 @@
+<display id="ucsc_vcf" version="1.0.0" name="display at UCSC">
+    <!-- Load links from file: one line to one link -->
+    <dynamic_links site_type="ucsc" skip_startswith="#" id="0" name="0">
+        <!-- Define parameters by column from file, allow splitting on builds -->
+        <dynamic_param name="site_id" value="0"/>
+        <dynamic_param name="ucsc_link" value="1"/>
+        <dynamic_param name="builds" value="2" split="True" separator="," />
+        <!-- Filter out some of the links based upon matching site_id to a Galaxy application configuration parameter and by dataset dbkey -->
+        <filter>${site_id in $APP.datatypes_registry.get_display_sites('ucsc')}</filter>
+        <filter>${dataset.dbkey in $builds}</filter>
+        <!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
+        <url>${ucsc_link}db=${qp($bgzip_file.dbkey)}&amp;hgt.customText=${qp($track.url)}</url>
+        <param type="data" name="bgzip_file" url="galaxy_${DATASET_HASH}.vcf.gz" format="vcf_bgzip" />
+        <param type="data" name="tabix_file" dataset="bgzip_file" url="galaxy_${DATASET_HASH}.vcf.gz.tbi" format="tabix" />
+        <param type="template" name="track" viewable="True">track type="vcfTabix" name="${bgzip_file.name.replace( '\\', '\\\\' ).replace( '"', '\\"' )}" bigDataUrl="${bgzip_file.url}" db="${bgzip_file.dbkey}"</param>
+    </dynamic_links>
+</display>

lib/galaxy/datatypes/images.py

@@ -5,8 +5,10 @@ import base64
 import json
 import logging
 import zipfile
+from io import StringIO
 from urllib.parse import quote_plus
 
+import mrcfile
 import numpy as np
 import tifffile
 
@@ -313,6 +315,35 @@ class Trk(Binary):
         return False
 
 
+class Mrc2014(Binary):
+    """
+    MRC/CCP4 2014 file format (.mrc).
+    https://www.ccpem.ac.uk/mrc_format/mrc2014.php
+
+    >>> from galaxy.datatypes.sniff import get_test_fname
+    >>> fname = get_test_fname('1.mrc')
+    >>> Mrc2014().sniff(fname)
+    True
+    >>> fname = get_test_fname('2.txt')
+    >>> Mrc2014().sniff(fname)
+    False
+    """
+    file_ext = 'mrc'
+
+    def sniff(self, filename):
+        # Handle the wierdness of mrcfile:
+        # https://github.com/ccpem/mrcfile/blob/master/mrcfile/validator.py#L88
+        try:
+            # An exception is thrown
+            # if the file is not an
+            # mrc2014 file.
+            if mrcfile.validate(filename, print_file=StringIO()):
+                return True
+        except Exception:
+            return False
+        return False
+
+
 class Gmaj(data.Data):
     """Class describing a GMAJ Applet"""
     edam_format = "format_3547"

lib/galaxy/dependencies/conditional-requirements.txt

@@ -14,7 +14,7 @@ python-pam
 galaxycloudrunner
 
 # For file sources plugins
-fs.webdavfs  # type: webdav
+fs.webdavfs>=0.4.2  # type: webdav
 fs.dropboxfs  # type: dropbox
 fs.sshfs  # type: ssh
 fs-s3fs  # type: s3

lib/galaxy/dependencies/dev-requirements.txt

@@ -3,7 +3,7 @@
 adal==1.2.6
 aiofiles==0.6.0
 alabaster==0.7.12; python_version >= "3.5"
-amqp==5.0.5; python_version >= "3.6"
+amqp==5.0.6; python_version >= "3.6"
 appdirs==1.4.4; python_version >= "3.6"
 argcomplete==1.12.2; python_version >= "3.6" and python_version < "4"
 async-exit-stack==1.0.1; python_version >= "3.6" and python_version < "3.7"
@@ -15,8 +15,9 @@ babel==2.9.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (py
 bagit-profile==1.3.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4"
 bagit==1.8.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6"
 bcrypt==3.2.0; python_version >= "3.6"
-bdbag==1.6.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0" and python_version < "4")
+bdbag==1.6.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0" and python_version < "4")
 beaker==1.11.0
+billiard==3.6.4.0; python_version >= "3.6"
 bioblend==0.15.0; python_version >= "3.6"
 bleach==3.3.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
 boltons==20.2.1
@@ -27,11 +28,16 @@ bx-python==0.8.11; python_version >= "3.6"
 cachecontrol==0.11.7; python_version >= "3.6" and python_version < "4"
 cached-property==1.5.2; python_version < "3.8" and python_version >= "3.6"
 cachetools==4.2.1; python_version >= "3.5" and python_version < "4.0" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0")
+celery==5.0.5; python_version >= "3.6"
 certifi==2020.12.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6"
-cffi==1.14.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+cffi==1.14.5; implementation_name == "pypy" and python_version >= "3.6" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0")
 chardet==4.0.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4"
 cheetah3==3.2.6.post1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
-click==7.1.2; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
+circus==0.17.1
+click-didyoumean==0.0.3; python_version >= "3.6"
+click-plugins==1.1.1; python_version >= "3.6"
+click-repl==0.1.6; python_version >= "3.6"
+click==7.1.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 cliff==3.7.0; python_version >= "3.6"
 cloudauthz==0.6.0
 cloudbridge==2.1.0
@@ -41,11 +47,11 @@ coloredlogs==15.0; python_version >= "3.6" and python_full_version < "3.0.0" and
 commonmark==0.9.1
 contextvars==2.4; python_version < "3.7" and python_version >= "3.6"
 coverage==5.5; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4"
-cryptography==3.4.6; python_version >= "3.6"
+cryptography==3.4.7; python_version >= "3.6"
 cwltool==3.0.20201109103151; python_version >= "3.6" and python_version < "4"
 dataclasses==0.8; python_version >= "3.6" and python_version < "3.7"
 debtcollector==2.2.0; python_version >= "3.6"
-decorator==4.4.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.2.0" and python_version >= "3.6"
+decorator==5.0.3; python_version >= "3.6"
 defusedxml==0.7.1; python_version >= "3.0" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.0"
 deprecated==1.2.12; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 deprecation==2.1.0
@@ -61,23 +67,23 @@ fluent-logger==0.10.0; python_version >= "3.5"
 funcsigs==1.0.2
 future==0.18.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0")
 galaxy-sequence-utils==1.1.5
-google-api-core==1.26.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
+google-api-core==1.26.3; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
 google-api-python-client==1.12.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
 google-auth-httplib2==0.1.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
 google-auth==1.28.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
 googleapis-common-protos==1.53.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
-gunicorn==20.0.4; python_version >= "3.4"
+gunicorn==20.1.0; python_version >= "3.5"
 gxformat2==0.15.0
 h11==0.12.0; python_version >= "3.6"
 h5py==3.1.0; python_version >= "3.6"
 httpcore==0.12.3; python_version >= "3.6"
-httplib2==0.19.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
+httplib2==0.19.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
 httpx==0.17.1; python_version >= "3.6"
 humanfriendly==9.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0"
 idna==2.10; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6"
 imagesize==1.2.0; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.5"
 immutables==0.15; python_version >= "3.6" and python_version < "3.7"
-importlib-metadata==3.7.3; python_version < "3.8" and python_version >= "3.6" and (python_version == "3.6" or python_version == "3.7") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.4.0" and python_version >= "3.6" and python_version < "3.8")
+importlib-metadata==3.10.0; python_version < "3.8" and python_version >= "3.6" and (python_version == "3.6" or python_version == "3.7") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.4.0" and python_version >= "3.6" and python_version < "3.8")
 importlib-resources==5.1.2; python_version < "3.7" and python_version >= "3.6"
 iniconfig==1.1.1; python_version >= "3.6"
 isa-rwval==0.10.10
@@ -90,9 +96,9 @@ jsonpointer==2.1; python_version >= "3.6" and python_full_version < "3.0.0" or p
 jsonschema==3.2.0
 keystoneauth1==4.3.1; python_version >= "3.6"
 kombu==5.0.2; python_version >= "3.6"
-lagom==1.1.0; python_version >= "3.6" and python_version < "4.0"
+lagom==1.2.0; python_version >= "3.6" and python_version < "4.0"
 lockfile==0.12.2; python_version >= "3.6" and python_version < "4"
-lxml==4.6.2; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
+lxml==4.6.3; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
 mako==1.1.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
 markdown-it-reporter==0.0.2
 markdown==3.3.4; python_version >= "3.6"
@@ -100,6 +106,7 @@ markupsafe==1.1.1; (python_version >= "2.7" and python_full_version < "3.0.0") o
 mercurial==5.7.1
 mirakuru==2.3.0; python_version >= "3.6"
 mistune==0.8.4; python_version >= "3.6" and python_version < "4"
+mrcfile==1.2.0; python_version >= "3.6"
 msgpack==1.0.2; python_version >= "3.6"
 munch==2.5.0; python_version >= "3.6"
 mypy-extensions==0.4.3; python_version >= "3.6" and python_version < "4"
@@ -132,14 +139,15 @@ pbr==5.5.1; python_version >= "3.6"
 pluggy==0.13.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 port-for==0.4; python_version >= "3.6"
 prettytable==0.7.2; python_version >= "3.6"
-protobuf==3.15.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+prompt-toolkit==3.0.3; python_version >= "3.6"
+protobuf==3.15.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 prov==1.5.1; python_version >= "3.6" and python_version < "4"
 psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
 pulsar-galaxy-lib==0.14.2
-py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or python_full_version >= "3.4.0" and python_version >= "3.6" and implementation_name == "pypy"
 pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
 pyasn1==0.4.8; python_version >= "3.5" and python_version < "4"
-pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.6" and python_full_version >= "3.4.0"
 pycryptodome==3.10.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
 pydantic==1.7.3; python_version >= "3.6" and python_version < "4.0"
 pydot==1.4.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.4.0"
@@ -178,6 +186,7 @@ python3-openid==3.2.0; python_version >= "3.0"
 pytz==2021.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6"
 pyuwsgi==2.0.19.1.post0
 pyyaml==5.4.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
+pyzmq==22.0.3; python_version >= "3.6"
 rdflib-jsonld==0.5.0; python_version >= "3.6" and python_version < "4"
 rdflib==4.2.2; python_version >= "3.6" and python_version < "4"
 recommonmark==0.7.1
@@ -187,14 +196,14 @@ requests-oauthlib==1.3.0; python_version >= "2.7" and python_full_version < "3.0
 requests-toolbelt==0.9.1; python_version >= "3.6"
 requests==2.25.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
 requestsexceptions==1.4.0; python_version >= "3.6"
-responses==0.13.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
+responses==0.13.2; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
 rfc3986==1.4.0; python_version >= "3.6"
 routes==2.5.1
 rsa==4.7.2; python_version >= "3.5" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
 ruamel.yaml.clib==0.2.2; platform_python_implementation == "CPython" and python_version < "3.8" and python_version >= "3.6"
 ruamel.yaml==0.16.5; python_version >= "3.6" and python_version < "4"
-s3transfer==0.3.4
-schema-salad==7.1.20210309094900; python_version >= "3.6" and python_version < "4"
+s3transfer==0.3.6
+schema-salad==7.1.20210316164414; python_version >= "3.6" and python_version < "4"
 selenium==3.141.0
 setuptools-scm==5.0.2; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4"
 shellescape==3.4.1; python_version >= "3.6" and python_version < "4"
@@ -206,7 +215,7 @@ social-auth-core==3.3.0
 sortedcontainers==2.3.0
 sphinx-markdown-tables==0.0.15
 sphinx-rtd-theme==0.5.1
-sphinx==3.5.2; python_version >= "3.5"
+sphinx==3.5.3; python_version >= "3.5"
 sphinxcontrib-applehelp==1.0.2; python_version >= "3.5"
 sphinxcontrib-devhelp==1.0.2; python_version >= "3.5"
 sphinxcontrib-htmlhelp==1.0.3; python_version >= "3.5"
@@ -214,8 +223,7 @@ sphinxcontrib-jsmath==1.0.1; python_version >= "3.5"
 sphinxcontrib-qthelp==1.0.3; python_version >= "3.5"
 sphinxcontrib-serializinghtml==1.1.4; python_version >= "3.5"
 sqlalchemy-migrate==0.13.0
-sqlalchemy-utils==0.36.7
-sqlalchemy==1.3.23; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
+sqlalchemy==1.3.24; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
 sqlitedict==1.7.0
 sqlparse==0.4.1; python_version >= "3.5"
 starlette-context==0.3.1; python_version >= "3.7"
@@ -228,6 +236,7 @@ tenacity==7.0.0
 testfixtures==6.17.1
 tifffile==2020.9.3
 toml==0.10.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6"
+tornado==6.1; python_version >= "3.5"
 tqdm==4.59.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
 twill==3.0
 typing-extensions==3.7.4.3; python_version >= "3.6" and python_version < "3.8"
@@ -245,6 +254,6 @@ webob==1.8.7; (python_version >= "2.7" and python_full_version < "3.0.0") or (py
 whoosh==2.7.4
 wrapt==1.12.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 xmlrunner==1.7.7
-yacman==0.8.0
+yacman==0.8.1
 zipp==3.4.1; python_version < "3.7" and python_version >= "3.6"
 zipstream-new==1.1.8

lib/galaxy/dependencies/pinned-lint-requirements.txt

@@ -1,13 +1,13 @@
 attrs==20.3.0
 flake8==3.9.0
-flake8-bugbear==21.3.2
+flake8-bugbear==21.4.3
 flake8-import-order==0.18.1
-importlib-metadata==3.7.3
+importlib-metadata==3.10.0
 mccabe==0.6.1
 mypy==0.790
 mypy-extensions==0.4.3
 pycodestyle==2.7.0
-pyflakes==2.3.0
+pyflakes==2.3.1
 typed-ast==1.4.2
 typing-extensions==3.7.4.3
 zipp==3.4.1

lib/galaxy/dependencies/pinned-requirements.txt

@@ -2,7 +2,7 @@
 
 adal==1.2.6
 aiofiles==0.6.0
-amqp==5.0.5; python_version >= "3.6"
+amqp==5.0.6; python_version >= "3.6"
 appdirs==1.4.4; python_version >= "3.6"
 argcomplete==1.12.2; python_version >= "3.6" and python_version < "4"
 async-exit-stack==1.0.1; python_version >= "3.6" and python_version < "3.7"
@@ -13,8 +13,9 @@ babel==2.9.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (py
 bagit-profile==1.3.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4"
 bagit==1.8.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6"
 bcrypt==3.2.0; python_version >= "3.6"
-bdbag==1.6.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0" and python_version < "4")
+bdbag==1.6.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0" and python_version < "4")
 beaker==1.11.0
+billiard==3.6.4.0; python_version >= "3.6"
 bioblend==0.15.0; python_version >= "3.6"
 bleach==3.3.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
 boltons==20.2.1
@@ -25,11 +26,16 @@ bx-python==0.8.11; python_version >= "3.6"
 cachecontrol==0.11.7; python_version >= "3.6" and python_version < "4"
 cached-property==1.5.2; python_version < "3.8" and python_version >= "3.6"
 cachetools==4.2.1; python_version >= "3.5" and python_version < "4.0" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0")
+celery==5.0.5; python_version >= "3.6"
 certifi==2020.12.5; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4"
-cffi==1.14.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+cffi==1.14.5; implementation_name == "pypy" and python_version >= "3.6" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0")
 chardet==4.0.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4"
 cheetah3==3.2.6.post1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
-click==7.1.2; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
+circus==0.17.1
+click-didyoumean==0.0.3; python_version >= "3.6"
+click-plugins==1.1.1; python_version >= "3.6"
+click-repl==0.1.6; python_version >= "3.6"
+click==7.1.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 cliff==3.7.0; python_version >= "3.6"
 cloudauthz==0.6.0
 cloudbridge==2.1.0
@@ -37,11 +43,11 @@ cmd2==1.5.0; python_version >= "3.6"
 colorama==0.4.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 coloredlogs==15.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0"
 contextvars==2.4; python_version >= "3.6" and python_version < "3.7"
-cryptography==3.4.6; python_version >= "3.6"
+cryptography==3.4.7; python_version >= "3.6"
 cwltool==3.0.20201109103151; python_version >= "3.6" and python_version < "4"
 dataclasses==0.8; python_version >= "3.6" and python_version < "3.7"
 debtcollector==2.2.0; python_version >= "3.6"
-decorator==4.4.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.2.0" and python_version >= "3.6"
+decorator==5.0.3; python_version >= "3.6"
 defusedxml==0.7.1; python_version >= "3.0" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.0"
 deprecation==2.1.0
 dictobj==0.4
@@ -55,7 +61,7 @@ fastapi==0.63.0; python_version >= "3.6"
 funcsigs==1.0.2
 future==0.18.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0")
 galaxy-sequence-utils==1.1.5
-google-api-core==1.26.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
+google-api-core==1.26.3; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
 google-api-python-client==1.12.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
 google-auth-httplib2==0.1.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
 google-auth==1.28.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
@@ -63,11 +69,11 @@ googleapis-common-protos==1.53.0; python_version >= "3.6" and python_full_versio
 gxformat2==0.15.0
 h11==0.12.0; python_version >= "3.6"
 h5py==3.1.0; python_version >= "3.6"
-httplib2==0.19.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
+httplib2==0.19.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
 humanfriendly==9.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0"
 idna==2.10; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6"
 immutables==0.15; python_version >= "3.6" and python_version < "3.7"
-importlib-metadata==3.7.3; python_version < "3.8" and python_version >= "3.6" and (python_version == "3.6" or python_version == "3.7")
+importlib-metadata==3.10.0; python_version < "3.8" and python_version >= "3.6" and (python_version == "3.6" or python_version == "3.7")
 importlib-resources==5.1.2; python_version < "3.7" and python_version >= "3.6"
 isa-rwval==0.10.10
 iso8601==0.1.14; python_version >= "3.6"
@@ -78,14 +84,15 @@ jsonpointer==2.1; python_version >= "3.6" and python_full_version < "3.0.0" or p
 jsonschema==3.2.0
 keystoneauth1==4.3.1; python_version >= "3.6"
 kombu==5.0.2; python_version >= "3.6"
-lagom==1.1.0; python_version >= "3.6" and python_version < "4.0"
+lagom==1.2.0; python_version >= "3.6" and python_version < "4.0"
 lockfile==0.12.2; python_version >= "3.6" and python_version < "4"
-lxml==4.6.2; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
+lxml==4.6.3; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
 mako==1.1.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
 markdown==3.3.4; python_version >= "3.6"
 markupsafe==1.1.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
 mercurial==5.7.1
 mistune==0.8.4; python_version >= "3.6" and python_version < "4"
+mrcfile==1.2.0; python_version >= "3.6"
 msgpack==1.0.2; python_version >= "3.6"
 munch==2.5.0; python_version >= "3.6"
 mypy-extensions==0.4.3; python_version >= "3.6" and python_version < "4"
@@ -115,13 +122,15 @@ paste==3.5.0
 pastedeploy==2.1.1
 pbr==5.5.1; python_version >= "3.6"
 prettytable==0.7.2; python_version >= "3.6"
-protobuf==3.15.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+prompt-toolkit==3.0.3; python_version >= "3.6"
+protobuf==3.15.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 prov==1.5.1; python_version >= "3.6" and python_version < "4"
 psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
 pulsar-galaxy-lib==0.14.2
+py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.6" and python_full_version >= "3.4.0"
 pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
 pyasn1==0.4.8; python_version >= "3.5" and python_version < "4"
-pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.6" and python_full_version >= "3.4.0"
 pycryptodome==3.10.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
 pydantic==1.7.3; python_version >= "3.6" and python_version < "4.0"
 pydot==1.4.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.4.0"
@@ -147,6 +156,7 @@ python3-openid==3.2.0; python_version >= "3.0"
 pytz==2021.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6"
 pyuwsgi==2.0.19.1.post0
 pyyaml==5.4.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
+pyzmq==22.0.3; python_version >= "3.6"
 rdflib-jsonld==0.5.0; python_version >= "3.6" and python_version < "4"
 rdflib==4.2.2; python_version >= "3.6" and python_version < "4"
 refgenconf==0.9.3
@@ -160,8 +170,8 @@ routes==2.5.1
 rsa==4.7.2; python_version >= "3.5" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
 ruamel.yaml.clib==0.2.2; platform_python_implementation == "CPython" and python_version < "3.8" and python_version >= "3.6"
 ruamel.yaml==0.16.5; python_version >= "3.6" and python_version < "4"
-s3transfer==0.3.4
-schema-salad==7.1.20210309094900; python_version >= "3.6" and python_version < "4"
+s3transfer==0.3.6
+schema-salad==7.1.20210316164414; python_version >= "3.6" and python_version < "4"
 setuptools-scm==5.0.2; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4"
 shellescape==3.4.1; python_version >= "3.6" and python_version < "4"
 simplejson==3.17.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6"
@@ -169,8 +179,7 @@ six==1.15.0; python_version >= "3.6" and python_full_version < "3.0.0" and pytho
 social-auth-core==3.3.0
 sortedcontainers==2.3.0
 sqlalchemy-migrate==0.13.0
-sqlalchemy-utils==0.36.7
-sqlalchemy==1.3.23; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
+sqlalchemy==1.3.24; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
 sqlitedict==1.7.0
 sqlparse==0.4.1; python_version >= "3.5"
 starlette-context==0.3.1; python_version >= "3.7"
@@ -180,6 +189,7 @@ svgwrite==1.4.1; python_version >= "3.6"
 tempita==0.5.2
 tenacity==7.0.0
 tifffile==2020.9.3
+tornado==6.1; python_version >= "3.5"
 tqdm==4.59.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
 typing-extensions==3.7.4.3; python_version >= "3.6" and python_version < "3.8"
 tzlocal==2.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4"
@@ -194,6 +204,6 @@ webencodings==0.5.1; python_version >= "2.7" and python_full_version < "3.0.0" o
 webob==1.8.7; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0")
 whoosh==2.7.4
 wrapt==1.12.1; python_version >= "3.6"
-yacman==0.8.0
+yacman==0.8.1
 zipp==3.4.1; python_version < "3.7" and python_version >= "3.6"
 zipstream-new==1.1.8

lib/galaxy/files/__init__.py

@@ -243,6 +243,23 @@ class ProvidesUserFileSourcesUserContext:
         user = self.trans.user
         return user and user.extra_preferences or defaultdict(lambda: None)
 
+    @property
+    def role_names(self):
+        """The set of role names of this user."""
+        user = self.trans.user
+        return user and set([ura.role.name for ura in user.roles])
+
+    @property
+    def group_names(self):
+        """The set of group names to which this user belongs."""
+        user = self.trans.user
+        return user and set([ugr.group.name for ugr in user.groups])
+
+    @property
+    def is_admin(self):
+        """Whether this user is an administrator."""
+        return self.trans.user_is_admin
+
 
 class DictFileSourcesUserContext:
 
@@ -264,3 +281,15 @@ class DictFileSourcesUserContext:
     @property
     def preferences(self):
         return self._kwd.get("preferences")
+
+    @property
+    def role_names(self):
+        return self._kwd.get("role_names")
+
+    @property
+    def group_names(self):
+        return self._kwd.get("group_names")
+
+    @property
+    def is_admin(self):
+        return self._kwd.get("is_admin")

lib/galaxy/files/_schema.py

@@ -1,5 +1,8 @@
 from enum import Enum
-from typing import List
+from typing import (
+    List,
+    Optional,
+)
 
 from pydantic import (
     BaseModel,
@@ -62,6 +65,16 @@ class FilesSourcePlugin(BaseModel):
         description="Whether this files source plugin allows write access.",
         example=False,
     )
+    requires_roles: Optional[str] = Field(
+        None,
+        title="Requires roles",
+        description="Only users with the roles specified here can access this files source.",
+    )
+    requires_groups: Optional[str] = Field(
+        None,
+        title="Requires groups",
+        description="Only users belonging to the groups specified here can access this files source.",
+    )
 
     class Config:
         # This allows additional fields (that are not validated)

lib/galaxy/files/sources/__init__.py

@@ -1,7 +1,16 @@
 import abc
 import os
 import time
+from typing import Set
 
+from galaxy.exceptions import (
+    ConfigurationError,
+    ItemAccessibilityException,
+)
+from galaxy.util.bool_expressions import (
+    BooleanExpressionEvaluator,
+    TokenContainedEvaluator,
+)
 from galaxy.util.template import fill_template
 
 DEFAULT_SCHEME = "gxfiles"
@@ -24,6 +33,10 @@ class FilesSource(metaclass=abc.ABCMeta):
     def get_writable(self):
         """Return a boolean indicating if this target is writable."""
 
+    @abc.abstractmethod
+    def user_has_access(self, user_context) -> bool:
+        """Return a boolean indicating if the user can access the FileSource."""
+
     # TODO: off-by-default
     @abc.abstractmethod
     def list(self, source_path="/", recursive=False, user_context=None):
@@ -57,6 +70,16 @@ class BaseFilesSource(FilesSource):
     def get_writable(self):
         return self.writable
 
+    def user_has_access(self, user_context) -> bool:
+        return (
+            user_context is None
+            or user_context.is_admin
+            or (
+                self._user_has_required_roles(user_context)
+                and self._user_has_required_groups(user_context)
+            )
+        )
+
     def get_uri_root(self):
         prefix = self.get_prefix()
         scheme = self.get_scheme()
@@ -69,13 +92,16 @@ class BaseFilesSource(FilesSource):
         uri_root = self.get_uri_root()
         return uri_join(uri_root, path)
 
-    def _parse_common_config_opts(self, kwd):
+    def _parse_common_config_opts(self, kwd: dict):
         self._file_sources_config = kwd.pop("file_sources_config")
         self.id = kwd.pop("id")
         self.label = kwd.pop("label", None) or self.id
         self.doc = kwd.pop("doc", None)
         self.scheme = kwd.pop("scheme", DEFAULT_SCHEME)
         self.writable = kwd.pop("writable", DEFAULT_WRITABLE)
+        self.requires_roles = kwd.pop("requires_roles", None)
+        self.requires_groups = kwd.pop("requires_groups", None)
+        self._validate_security_rules()
         # If coming from to_dict, strip API helper values
         kwd.pop("uri_root", None)
         kwd.pop("type", None)
@@ -89,6 +115,8 @@ class BaseFilesSource(FilesSource):
             "label": self.label,
             "doc": self.doc,
             "writable": self.writable,
+            "requires_roles": self.requires_roles,
+            "requires_groups": self.requires_groups,
         }
         if for_serialization:
             rval.update(self._serialization_props(user_context=user_context))
@@ -109,15 +137,37 @@ class BaseFilesSource(FilesSource):
         Used in to_dict method if for_serialization is True.
         """
 
+    def list(self, path="/", recursive=False, user_context=None):
+        self._check_user_access(user_context)
+        return self._list(path, recursive, user_context)
+
+    @abc.abstractmethod
+    def _list(self, path="/", recursive=False, user_context=None):
+        pass
+
     def write_from(self, target_path, native_path, user_context=None):
         if not self.get_writable():
             raise Exception("Cannot write to a non-writable file source.")
+        self._check_user_access(user_context)
         self._write_from(target_path, native_path, user_context=user_context)
 
     @abc.abstractmethod
     def _write_from(self, target_path, native_path, user_context=None):
         pass
 
+    def realize_to(self, source_path, native_path, user_context=None):
+        self._check_user_access(user_context)
+        self._realize_to(source_path, native_path, user_context)
+
+    @abc.abstractmethod
+    def _realize_to(self, source_path, native_path, user_context=None):
+        pass
+
+    def _check_user_access(self, user_context):
+        """Raises an exception if the given user doesn't have the rights to access this file source."""
+        if user_context is not None and not self.user_has_access(user_context):
+            raise ItemAccessibilityException(f"User {user_context.username} has no access to file source.")
+
     def _evaluate_prop(self, prop_val, user_context):
         rval = prop_val
         if isinstance(prop_val, str) and "$" in prop_val:
@@ -130,6 +180,33 @@ class BaseFilesSource(FilesSource):
 
         return rval
 
+    def _user_has_required_roles(self, user_context) -> bool:
+        if self.requires_roles:
+            return self._evaluate_security_rules(self.requires_roles, user_context.role_names)
+        return True
+
+    def _user_has_required_groups(self, user_context) -> bool:
+        if self.requires_groups:
+            return self._evaluate_security_rules(self.requires_groups, user_context.group_names)
+        return True
+
+    def _evaluate_security_rules(self, rule_expression: str, user_credentials: Set[str]) -> bool:
+        token_evaluator = TokenContainedEvaluator(user_credentials)
+        evaluator = BooleanExpressionEvaluator(token_evaluator)
+        return evaluator.evaluate_expression(rule_expression)
+
+    def _validate_security_rules(self) -> None:
+        """Checks if the security rules defined in the plugin configuration are valid boolean expressions or raises
+        a ConfigurationError exception otherwise."""
+
+        def _get_error_msg_for(rule_name: str) -> str:
+            return f"Invalid boolean expression for '{rule_name}' in {self.label} file source plugin configuration."
+
+        if self.requires_roles and not BooleanExpressionEvaluator.is_valid_expression(self.requires_roles):
+            raise ConfigurationError(_get_error_msg_for("requires_roles"))
+        if self.requires_groups and not BooleanExpressionEvaluator.is_valid_expression(self.requires_groups):
+            raise ConfigurationError(_get_error_msg_for("requires_groups"))
+
 
 def uri_join(*args):
     # url_join doesn't work with non-standard scheme

lib/galaxy/files/sources/_pyfilesystem2.py

@@ -22,7 +22,7 @@ class PyFilesystem2FilesSource(BaseFilesSource):
     def _open_fs(self, user_context=None):
         """Subclasses must instantiate a PyFilesystem2 handle for this file system."""
 
-    def list(self, path="/", recursive=False, user_context=None):
+    def _list(self, path="/", recursive=False, user_context=None):
         """Return dictionary of 'Directory's and 'File's."""
 
         with self._open_fs(user_context=user_context) as h:
@@ -38,7 +38,7 @@ class PyFilesystem2FilesSource(BaseFilesSource):
                 to_dict = functools.partial(self._resource_info_to_dict, path)
                 return list(map(to_dict, res))
 
-    def realize_to(self, source_path, native_path, user_context=None):
+    def _realize_to(self, source_path, native_path, user_context=None):
         with open(native_path, 'wb') as write_file:
             self._open_fs(user_context=user_context).download(source_path, write_file)
 

lib/galaxy/files/sources/posix.py

@@ -30,7 +30,7 @@ class PosixFilesSource(BaseFilesSource):
         self.enforce_symlink_security = props.get("enforce_symlink_security", DEFAULT_ENFORCE_SYMLINK_SECURITY)
         self.delete_on_realize = props.get("delete_on_realize", DEFAULT_DELETE_ON_REALIZE)
 
-    def list(self, path="/", recursive=True, user_context=None):
+    def _list(self, path="/", recursive=True, user_context=None):
         dir_path = self._to_native_path(path, user_context=user_context)
         if not self._safe_directory(dir_path):
             raise exceptions.ObjectNotFound('The specified directory does not exist [%s].' % dir_path)
@@ -47,7 +47,7 @@ class PosixFilesSource(BaseFilesSource):
             to_dict = functools.partial(self._resource_info_to_dict, path, user_context=user_context)
             return list(map(to_dict, res))
 
-    def realize_to(self, source_path, native_path, user_context=None):
+    def _realize_to(self, source_path, native_path, user_context=None):
         effective_root = self._effective_root(user_context)
         source_native_path = self._to_native_path(source_path, user_context=user_context)
         if self.enforce_symlink_security:

lib/galaxy/files/sources/s3fs.py

@@ -1,4 +1,5 @@
 import functools
+import logging
 import os
 
 try:
@@ -11,6 +12,8 @@ from ..sources import BaseFilesSource
 DEFAULT_ENFORCE_SYMLINK_SECURITY = True
 DEFAULT_DELETE_ON_REALIZE = False
 
+log = logging.getLogger(__name__)
+
 
 class S3FsFilesSource(BaseFilesSource):
     plugin_type = 's3fs'
@@ -19,11 +22,12 @@ class S3FsFilesSource(BaseFilesSource):
         if s3fs is None:
             raise Exception("Package s3fs unavailable but required for this file source plugin.")
         props = self._parse_common_config_opts(kwd)
-        self._bucket = props.pop("bucket")
-        assert self._bucket
+        self._bucket = props.pop("bucket", '')
+        self._endpoint_url = props.pop('endpoint_url', None)
+        assert self._endpoint_url or self._bucket
         self._props = props
 
-    def list(self, path="/", recursive=True, user_context=None):
+    def _list(self, path="/", recursive=True, user_context=None):
         fs = self._open_fs(user_context=user_context)
         if recursive:
             res = []
@@ -39,7 +43,7 @@ class S3FsFilesSource(BaseFilesSource):
             to_dict = functools.partial(self._resource_info_to_dict, path)
             return list(map(to_dict, res))
 
-    def realize_to(self, source_path, native_path, user_context=None):
+    def _realize_to(self, source_path, native_path, user_context=None):
         bucket_path = self._bucket_path(source_path)
         self._open_fs(user_context=user_context).download(bucket_path, native_path)
 
@@ -52,6 +56,8 @@ class S3FsFilesSource(BaseFilesSource):
         return f"{self._bucket}{path}"
 
     def _open_fs(self, user_context=None):
+        if self._endpoint_url:
+            self._props.update({'client_kwargs': {'endpoint_url': self._endpoint_url}})
         fs = s3fs.S3FileSystem(**self._props)
         return fs
 

lib/galaxy/job_execution/output_collect.py

@@ -177,10 +177,24 @@ class BaseJobContext:
             filenames[discovered_file.path] = discovered_file
         return filenames
 
+    def get_job_id(self):
+        return None  # overwritten in subclasses
+
 
 class JobContext(ModelPersistenceContext, BaseJobContext):
 
-    def __init__(self, tool, tool_provided_metadata, job, job_working_directory, permission_provider, metadata_source_provider, input_dbkey, object_store, final_job_state):
+    def __init__(
+            self,
+            tool,
+            tool_provided_metadata,
+            job,
+            job_working_directory,
+            permission_provider,
+            metadata_source_provider,
+            input_dbkey,
+            object_store,
+            final_job_state,
+            flush_per_n_datasets=None):
         self.tool = tool
         self.metadata_source_provider = metadata_source_provider
         self.permission_provider = permission_provider
@@ -192,6 +206,7 @@ class JobContext(ModelPersistenceContext, BaseJobContext):
         self.tool_provided_metadata = tool_provided_metadata
         self.object_store = object_store
         self.final_job_state = final_job_state
+        self.flush_per_n_datasets = flush_per_n_datasets
 
     @property
     def work_context(self):
@@ -299,6 +314,9 @@ class JobContext(ModelPersistenceContext, BaseJobContext):
     def job_id(self):
         return self.job.id
 
+    def get_job_id(self):
+        return self.job.id
+
 
 class SessionlessJobContext(SessionlessModelPersistenceContext, BaseJobContext):
 
@@ -312,6 +330,7 @@ class SessionlessJobContext(SessionlessModelPersistenceContext, BaseJobContext):
         self.import_store = import_store
         self.input_dbkey = input_dbkey
         self.final_job_state = final_job_state
+        self.flush_per_n_datasets = None
 
     def output_collection_def(self, name):
         tool_as_dict = self.metadata_params["tool"]
@@ -355,8 +374,10 @@ class SessionlessJobContext(SessionlessModelPersistenceContext, BaseJobContext):
             self.export_store.collection_datasets[collection_dataset.id] = True
 
     def add_output_dataset_association(self, name, dataset_instance):
-        job_id = self.metadata_params["job_id_tag"]
-        self.export_store.add_job_output_dataset_associations(job_id, name, dataset_instance)
+        self.export_store.add_job_output_dataset_associations(self.get_job_id(), name, dataset_instance)
+
+    def get_job_id(self):
+        return self.metadata_params["job_id_tag"]
 
 
 def collect_primary_datasets(job_context, output, input_ext):
@@ -419,6 +440,7 @@ def collect_primary_datasets(job_context, output, input_ext):
                 info=info,
                 init_from=outdata,
                 dataset_attributes=new_primary_datasets_attributes,
+                creating_job_id=job_context.get_job_id() if job_context else None
             )
             # Associate new dataset with job
             job_context.add_output_dataset_association(f'__new_primary_file_{name}|{designation}__', primary_data)
@@ -478,10 +500,11 @@ def walk_over_extra_files(target_dir, extra_file_collector, job_working_director
     if os.path.isdir(directory):
         for filename in os.listdir(directory):
             path = os.path.join(directory, filename)
-            if os.path.isdir(path) and extra_file_collector.recurse:
-                # The current directory is already validated, so use that as the next job_working_directory when recursing
-                for match in walk_over_extra_files(filename, extra_file_collector, directory, matchable):
-                    yield match
+            if os.path.isdir(path):
+                if extra_file_collector.recurse:
+                    # The current directory is already validated, so use that as the next job_working_directory when recursing
+                    for match in walk_over_extra_files(filename, extra_file_collector, directory, matchable):
+                        yield match
             else:
                 match = extra_file_collector.match(matchable, filename, path=path)
                 if match:

lib/galaxy/jobs/__init__.py

@@ -57,7 +57,7 @@ from galaxy.jobs.runners import BaseJobRunner, JobState
 from galaxy.metadata import get_metadata_compute_strategy
 from galaxy.model import store
 from galaxy.objectstore import ObjectStorePopulator
-from galaxy.structured_app import StructuredApp
+from galaxy.structured_app import MinimalManagerApp
 from galaxy.tool_util.deps import requirements
 from galaxy.tool_util.output_checker import (
     check_output,
@@ -301,7 +301,7 @@ class JobConfiguration(ConfiguresHandlers):
         <when value="yes"/>
     </conditional>"""
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         """Parse the job configuration XML.
         """
         self.app = app
@@ -1348,7 +1348,10 @@ class JobWrapper(HasResourceParameters):
             ActionBox.execute(self.app, self.sa_session, pja, job)
         # If the job was deleted, call tool specific fail actions (used for e.g. external metadata) and clean up
         if self.tool:
-            self.tool.job_failed(self, message, exception)
+            try:
+                self.tool.job_failed(self, message, exception)
+            except Exception:
+                log.exception(f"Error occured while calling tool specific fail actions for job {job.id}")
         cleanup_job = self.cleanup_job
         delete_files = cleanup_job == 'always' or (cleanup_job == 'onsuccess' and job.state == job.states.DELETED)
         self.cleanup(delete_files=delete_files)

lib/galaxy/jobs/manager.py

@@ -9,7 +9,7 @@ from sqlalchemy.sql.expression import null
 from galaxy.exceptions import HandlerAssignmentError, ToolExecutionError
 from galaxy.jobs import handler, NoopQueue
 from galaxy.model import Job
-from galaxy.structured_app import StructuredApp
+from galaxy.structured_app import MinimalManagerApp
 from galaxy.web_stack.message import JobHandlerMessage
 
 log = logging.getLogger(__name__)
@@ -21,7 +21,7 @@ class JobManager:
     """
     job_handler: handler.JobHandlerI
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         self.app = app
         self.job_lock = False
         if self.app.is_job_handler:

lib/galaxy/jobs/runners/__init__.py

@@ -141,7 +141,9 @@ class BaseJobRunner:
                     job_state = JobState(job_wrapper=arg, job_destination={})
                 else:
                     job_state = arg
-                self.work_queue.put((self.fail_job, job_state))
+                if method != self.fail_job:
+                    # Prevent fail_job cycle in the work_queue
+                    self.work_queue.put((self.fail_job, job_state))
 
     # Causes a runner's `queue_job` method to be called from a worker thread
     def put(self, job_wrapper):

lib/galaxy/jobs/runners/kubernetes.py

@@ -6,6 +6,7 @@ import logging
 import math
 import os
 import re
+from datetime import datetime
 
 import yaml
 
@@ -21,6 +22,7 @@ from galaxy.jobs.runners.util.pykube_util import (
     find_job_object_by_name,
     find_pod_object_by_name,
     galaxy_instance_id,
+    is_pod_unschedulable,
     Job,
     job_object_dict,
     Pod,
@@ -70,7 +72,8 @@ class KubernetesJobRunner(AsynchronousJobRunner):
             k8s_fs_group_id=dict(map=int),
             k8s_cleanup_job=dict(map=str, valid=lambda s: s in {"onsuccess", "always", "never"}, default="always"),
             k8s_pod_retries=dict(map=int, valid=lambda x: int(x) >= 0, default=3),
-            k8s_walltime_limit=dict(map=int, valid=lambda x: int(x) >= 0, default=172800))
+            k8s_walltime_limit=dict(map=int, valid=lambda x: int(x) >= 0, default=172800),
+            k8s_unschedulable_walltime_limit=dict(map=int, valid=lambda x: int(x) >= 0, default=1800),)
 
         if 'runner_param_specs' not in kwargs:
             kwargs['runner_param_specs'] = dict()
@@ -484,8 +487,15 @@ class KubernetesJobRunner(AsynchronousJobRunner):
                 return None
             elif active > 0 and failed <= max_pod_retries:
                 if not job_state.running:
-                    job_state.running = True
-                    job_state.job_wrapper.change_state(model.Job.states.RUNNING)
+                    if self.__job_pending_due_to_unschedulable_pod(job_state):
+                        creation_time_str = job.obj['metadata'].get('creationTimestamp')
+                        creation_time = datetime.strptime(creation_time_str, '%Y-%m-%dT%H:%M:%SZ')
+                        elapsed_seconds = (datetime.utcnow() - creation_time).total_seconds()
+                        if elapsed_seconds > self.runner_params['k8s_unschedulable_walltime_limit']:
+                            return self._handle_unschedulable_job(job, job_state)
+                    else:
+                        job_state.running = True
+                        job_state.job_wrapper.change_state(model.Job.states.RUNNING)
                 return job_state
             elif job_persisted_state == model.Job.states.DELETED:
                 # Job has been deleted via stop_job and job has not been deleted,
@@ -515,6 +525,18 @@ class KubernetesJobRunner(AsynchronousJobRunner):
             # job is no longer viable - remove from watched jobs
             return None
 
+    def _handle_unschedulable_job(self, job, job_state):
+        # Handle unschedulable job that exceeded deadline
+        job_state.fail_message = "Job was unschedulable longer than specified deadline"
+        job_state.runner_state = JobState.runner_states.WALLTIME_REACHED
+        job_state.running = False
+        self.mark_as_failed(job_state)
+        try:
+            self.__cleanup_k8s_job(job)
+        except Exception:
+            log.exception("Could not clean up k8s batch job. Ignoring...")
+        return None
+
     def _handle_job_failure(self, job, job_state):
         # Figure out why job has failed
         with open(job_state.error_file, 'a') as error_file:
@@ -563,6 +585,17 @@ class KubernetesJobRunner(AsynchronousJobRunner):
 
         return False
 
+    def __job_pending_due_to_unschedulable_pod(self, job_state):
+        """
+        checks the state of the pod to see if it is unschedulable.
+        """
+        pods = find_pod_object_by_name(self._pykube_api, job_state.job_id, self.runner_params['k8s_namespace'])
+        if not pods.response['items']:
+            return False
+
+        pod = Pod(self._pykube_api, pods.response['items'][0])
+        return is_pod_unschedulable(self._pykube_api, pod, self.runner_params['k8s_namespace'])
+
     def stop_job(self, job_wrapper):
         """Attempts to delete a dispatched job to the k8s cluster"""
         job = job_wrapper.get_job()

lib/galaxy/jobs/runners/pulsar.py

@@ -305,7 +305,7 @@ class PulsarJobRunner(AsynchronousJobRunner):
 
     def _update_job_state_for_status(self, job_state, pulsar_status, full_status=None):
         log.debug('(%s) Received status update: %s %s', job_state.job_id, type(pulsar_status), pulsar_status)
-        if pulsar_status == "complete" or job_state.job_wrapper.get_state() == model.Job.states.STOPPED:
+        if pulsar_status in ["complete", "cancelled"] or job_state.job_wrapper.get_state() == model.Job.states.STOPPED:
             self.mark_as_finished(job_state)
             return None
         if pulsar_status in ["failed", "lost"]:
@@ -382,6 +382,7 @@ class PulsarJobRunner(AsynchronousJobRunner):
             if os.path.exists(tool_script):
                 log.debug("Registering tool_script for Pulsar transfer [%s]" % tool_script)
                 job_directory_files.append(tool_script)
+                config_files.append(tool_script)
             # Following is job destination environment variables
             env = client.env
             # extend it with tool defined environment variables

lib/galaxy/jobs/runners/state_handlers/resubmit.py

@@ -187,7 +187,7 @@ class _ExpressionContext:
             self._lazy_context = {
                 "walltime_reached": runner_state == JobState.runner_states.WALLTIME_REACHED,
                 "memory_limit_reached": runner_state == JobState.runner_states.MEMORY_LIMIT_REACHED,
-                "unknown_error": JobState.runner_states.UNKNOWN_ERROR,
+                "unknown_error": runner_state == JobState.runner_states.UNKNOWN_ERROR,
                 "tool_detected_failure": runner_state == JobState.runner_states.TOOL_DETECT_ERROR,
                 "any_failure": True,
                 "any_potential_job_failure": True,  # Add a hook here - later on allow tools to describe things that are definitely input problems.

lib/galaxy/jobs/runners/util/pykube_util.py

@@ -68,6 +68,14 @@ def find_pod_object_by_name(pykube_api, job_name, namespace=None):
     return Pod.objects(pykube_api).filter(selector="job-name=" + job_name, namespace=namespace)
 
 
+def is_pod_unschedulable(pykube_api, pod, namespace=None):
+    is_unschedulable = bool(len([c for c in pod.obj['status']['conditions'] if c.get("reason") == "Unschedulable"]))
+    if pod.obj['status']['phase'] == "Pending" and is_unschedulable:
+        return True
+
+    return False
+
+
 def stop_job(job, cleanup="always"):
     job_failed = (job.obj['status']['failed'] > 0
                   if 'failed' in job.obj['status'] else False)
@@ -126,6 +134,7 @@ __all__ = (
     "find_job_object_by_name",
     "find_pod_object_by_name",
     "galaxy_instance_id",
+    "is_pod_unschedulable",
     "Job",
     "job_object_dict",
     "Pod",

lib/galaxy/managers/base.py

@@ -37,7 +37,7 @@ from sqlalchemy.orm.scoping import scoped_session
 from galaxy import exceptions
 from galaxy import model
 from galaxy.model import tool_shed_install
-from galaxy.structured_app import BasicApp, StructuredApp
+from galaxy.structured_app import BasicApp, MinimalManagerApp
 from galaxy.util import namedtuple
 
 log = logging.getLogger(__name__)
@@ -488,7 +488,7 @@ class HasAModelManager:
     # examples where this doesn't really work are ConfigurationSerializer (no manager)
     # and contents (2 managers)
 
-    def __init__(self, app: StructuredApp, manager=None, **kwargs):
+    def __init__(self, app: MinimalManagerApp, manager=None, **kwargs):
         self._manager = manager
 
     @property
@@ -542,7 +542,7 @@ class ModelSerializer(HasAModelManager):
     default_view: Optional[str]
     views: Dict[str, List[str]]
 
-    def __init__(self, app: StructuredApp, **kwargs):
+    def __init__(self, app: MinimalManagerApp, **kwargs):
         """
         Set up serializer map, any additional serializable keys, and views here.
         """
@@ -713,7 +713,7 @@ class ModelDeserializer(HasAModelManager):
     """
     # TODO:?? a larger question is: which should be first? Deserialize then validate - or - validate then deserialize?
 
-    def __init__(self, app: StructuredApp, validator=None, **kwargs):
+    def __init__(self, app: MinimalManagerApp, validator=None, **kwargs):
         """
         Set up deserializers and validator.
         """
@@ -904,7 +904,7 @@ class ModelFilterParser(HasAModelManager):
     orm_filter_parsers: Dict[str, Dict]
     fn_filter_parsers: Dict[str, Dict]
 
-    def __init__(self, app: StructuredApp, **kwargs):
+    def __init__(self, app: MinimalManagerApp, **kwargs):
         """
         Set up serializer map, any additional serializable keys, and views here.
         """

lib/galaxy/managers/collections.py

@@ -254,18 +254,6 @@ class DatasetCollectionManager:
         for _, tag in tags.items():
             dataset_collection_instance.tags.append(tag.copy(cls=model.HistoryDatasetCollectionTagAssociation))
 
-    def set_collection_elements(self, dataset_collection, dataset_instances):
-        if dataset_collection.populated:
-            raise Exception("Cannot reset elements of an already populated dataset collection.")
-
-        collection_type = dataset_collection.collection_type
-        collection_type_description = self.collection_type_descriptions.for_collection_type(collection_type)
-        type_plugin = collection_type_description.rank_type_plugin()
-        builder.set_collection_elements(dataset_collection, type_plugin, dataset_instances)
-        dataset_collection.mark_as_populated()
-
-        return dataset_collection
-
     def collection_builder_for(self, dataset_collection):
         return builder.BoundCollectionBuilder(dataset_collection)
 
@@ -299,7 +287,7 @@ class DatasetCollectionManager:
         changed = self._set_from_dict(trans, dataset_collection_instance, payload)
         return changed
 
-    def copy(self, trans, parent, source, encoded_source_id, copy_elements=False):
+    def copy(self, trans, parent, source, encoded_source_id, copy_elements=False, dataset_instance_attributes=None):
         """
         PRECONDITION: security checks on ability to add to parent occurred
         during load.
@@ -308,7 +296,9 @@ class DatasetCollectionManager:
         source_hdca = self.__get_history_collection_instance(trans, encoded_source_id)
         copy_kwds = {}
         if copy_elements:
-            copy_kwds["element_destination"] = parent
+            copy_kwds["element_destination"] = parent  # e.g. a history
+        if dataset_instance_attributes is not None:
+            copy_kwds["dataset_instance_attributes"] = dataset_instance_attributes
         new_hdca = source_hdca.copy(**copy_kwds)
         new_hdca.copy_tags_from(target_user=trans.get_user(), source=source_hdca)
         if not copy_elements:

lib/galaxy/managers/configuration.py

@@ -15,7 +15,7 @@ from typing import (
     List,
 )
 
-from galaxy.app import StructuredApp
+from galaxy.app import MinimalManagerApp
 from galaxy.managers import base
 from galaxy.managers.context import ProvidesUserContext
 from galaxy.schema.fields import EncodedDatabaseIdField
@@ -30,7 +30,7 @@ VERSION_JSON_FILE = 'version.json'
 class ConfigurationManager:
     """Interface/service object for interacting with configuration and related data."""
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         self._app = app
 
     def get_configuration(
@@ -122,6 +122,9 @@ class ConfigSerializer(base.ModelSerializer):
             assert hasattr(config, key)
             return getattr(config, key)
 
+        def _config_is_truthy(config, key, **context):
+            return True if config.get(key) else False
+
         self.serializers = {
             # TODO: this is available from user data, remove
             'is_admin_user': lambda *a, **c: False,
@@ -151,6 +154,7 @@ class ConfigSerializer(base.ModelSerializer):
             'allow_user_impersonation': _use_config,
             'allow_user_creation': _defaults_to(False),  # schema default is True
             'use_remote_user': _defaults_to(None),  # schema default is False; or config.single_user
+            'single_user': _config_is_truthy,
             'enable_oidc': _use_config,
             'oidc': _use_config,
             'enable_quotas': _use_config,
@@ -160,7 +164,6 @@ class ConfigSerializer(base.ModelSerializer):
             'ga_code': _use_config,
             'enable_unique_workflow_defaults': _use_config,
             'enable_beta_markdown_export': _use_config,
-            'simplified_workflow_run_ui': _use_config,
             'simplified_workflow_run_ui_target_history': _use_config,
             'simplified_workflow_run_ui_job_cache': _use_config,
             'has_user_tool_filters': _defaults_to(False),

lib/galaxy/managers/context.py

@@ -50,7 +50,7 @@ from galaxy.model import (
 )
 from galaxy.model.base import ModelMapping
 from galaxy.security.idencoding import IdEncodingHelper
-from galaxy.structured_app import StructuredApp
+from galaxy.structured_app import MinimalManagerApp
 from galaxy.util import bunch
 
 
@@ -62,7 +62,7 @@ class ProvidesAppContext:
     """
 
     @abc.abstractproperty
-    def app(self) -> StructuredApp:
+    def app(self) -> MinimalManagerApp:
         """Provide access to the Galaxy ``app`` object.
         """
 

lib/galaxy/managers/datasets.py

@@ -18,7 +18,7 @@ from galaxy.managers import (
     secured,
     users
 )
-from galaxy.structured_app import StructuredApp
+from galaxy.structured_app import MinimalManagerApp
 from galaxy.util.checkers import check_binary
 
 log = logging.getLogger(__name__)
@@ -33,7 +33,7 @@ class DatasetManager(base.ModelManager, secured.AccessibleManagerMixin, deletabl
 
     # TODO:?? get + error_if_uploading is common pattern, should upload check be worked into access/owed?
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         super().__init__(app)
         self.permissions = DatasetRBACPermissions(app)
         # needed for admin test
@@ -143,7 +143,7 @@ class DatasetRBACPermissions:
 class DatasetSerializer(base.ModelSerializer, deletable.PurgableSerializerMixin):
     model_manager_class = DatasetManager
 
-    def __init__(self, app: StructuredApp, user_manager: users.UserManager):
+    def __init__(self, app: MinimalManagerApp, user_manager: users.UserManager):
         super().__init__(app)
         self.dataset_manager = self.manager
         # needed for admin test
@@ -302,8 +302,10 @@ class DatasetAssociationManager(base.ModelManager,
             if not job.finished:
                 # Are *all* of the job's other output datasets deleted?
                 if job.check_if_output_datasets_deleted():
-                    job.mark_deleted(self.app.config.track_jobs_in_database)
-                    self.app.job_manager.stop(job)
+                    track_jobs_in_database = self.app.config.track_jobs_in_database
+                    job.mark_deleted(track_jobs_in_database)
+                    if not track_jobs_in_database:
+                        self.app.job_manager.stop(job)
                     return True
         return False
 

lib/galaxy/managers/folders.py

@@ -8,6 +8,7 @@ from sqlalchemy.orm.exc import (
     NoResultFound
 )
 
+from galaxy import util
 from galaxy.exceptions import (
     AuthenticationRequired,
     InconsistentDatabase,
@@ -15,9 +16,11 @@ from galaxy.exceptions import (
     InternalServerError,
     ItemAccessibilityException,
     MalformedId,
-    RequestParameterInvalidException
+    RequestParameterInvalidException,
+    RequestParameterMissingException,
 )
-from galaxy.util import unicodify
+from galaxy.managers.roles import RoleManager
+from galaxy.structured_app import StructuredApp
 
 log = logging.getLogger(__name__)
 
@@ -51,7 +54,7 @@ class FolderManager:
         except NoResultFound:
             raise RequestParameterInvalidException('No folder found with the id provided.')
         except Exception as e:
-            raise InternalServerError('Error loading from the database.' + unicodify(e))
+            raise InternalServerError('Error loading from the database.' + util.unicodify(e))
         folder = self.secure(trans, folder, check_manageable, check_accessible)
         return folder
 
@@ -309,3 +312,260 @@ class FolderManager:
         :rtype:    int
         """
         return self.decode_folder_id(trans, self.cut_the_prefix(encoded_folder_id))
+
+
+class FoldersService:
+    """Common interface/service logic for interactions with library folders in the context of the API.
+    Provides the logic of the actions invoked by API controllers and uses type definitions
+    and pydantic models to declare its parameters and return types.
+    """
+
+    def __init__(self, app: StructuredApp, folder_manager: FolderManager, role_manager: RoleManager) -> None:
+        self._app = app
+        self.folder_manager = folder_manager
+        self.role_manager = role_manager
+
+    def show(self, trans, id):
+        """
+        Displays information about a folder.
+
+        :param  id:      the folder's encoded id (required)
+        :type   id:      an encoded id string (has to be prefixed by 'F')
+
+        :returns:   dictionary including details of the folder
+        :rtype:     dict
+        """
+        folder_id = self.folder_manager.cut_and_decode(trans, id)
+        folder = self.folder_manager.get(trans, folder_id, check_manageable=False, check_accessible=True)
+        return_dict = self.folder_manager.get_folder_dict(trans, folder)
+        return return_dict
+
+    def create(self, trans, encoded_parent_folder_id, payload: dict):
+        """
+        Create a new folder object underneath the one specified in the parameters.
+
+        :param  encoded_parent_folder_id:      (required) the parent folder's id
+        :type   encoded_parent_folder_id:      an encoded id string (should be prefixed by 'F')
+        :param   payload: dictionary structure containing:
+
+            :param  name:                          (required) the name of the new folder
+            :type   name:                          str
+            :param  description:                   the description of the new folder
+            :type   description:                   str
+
+        :type       dictionary
+        :returns:   information about newly created folder, notably including ID
+        :rtype:     dictionary
+        :raises: RequestParameterMissingException
+        """
+        name = payload.get('name', None)
+        if name is None:
+            raise RequestParameterMissingException("Missing required parameter 'name'.")
+        description = payload.get('description', '')
+        decoded_parent_folder_id = self.folder_manager.cut_and_decode(trans, encoded_parent_folder_id)
+        parent_folder = self.folder_manager.get(trans, decoded_parent_folder_id)
+        new_folder = self.folder_manager.create(trans, parent_folder.id, name, description)
+        return self.folder_manager.get_folder_dict(trans, new_folder)
+
+    def get_permissions(self, trans, encoded_folder_id, scope=None, page=None, page_limit=None, query=None):
+        """
+        Load all permissions for the given folder id and return it.
+
+        :param  encoded_folder_id:     the encoded id of the folder
+        :type   encoded_folder_id:     an encoded id string
+
+        :param  scope:      either 'current' or 'available'
+        :type   scope:      string
+
+        :returns:   dictionary with all applicable permissions' values
+        :rtype:     dictionary
+
+        :raises: InsufficientPermissionsException
+        """
+        current_user_roles = trans.get_current_user_roles()
+        is_admin = trans.user_is_admin
+        decoded_folder_id = self.folder_manager.cut_and_decode(trans, encoded_folder_id)
+        folder = self.folder_manager.get(trans, decoded_folder_id)
+
+        if not (is_admin or trans.app.security_agent.can_manage_library_item(current_user_roles, folder)):
+            raise InsufficientPermissionsException('You do not have proper permission to access permissions of this folder.')
+
+        if scope == 'current' or scope is None:
+            return self.folder_manager.get_current_roles(trans, folder)
+        #  Return roles that are available to select.
+        elif scope == 'available':
+            if page is not None:
+                page = int(page)
+            else:
+                page = 1
+            if page_limit is not None:
+                page_limit = int(page_limit)
+            else:
+                page_limit = 10
+            roles, total_roles = trans.app.security_agent.get_valid_roles(trans, folder, query, page, page_limit)
+            return_roles = []
+            for role in roles:
+                role_id = trans.security.encode_id(role.id)
+                return_roles.append(dict(id=role_id, name=role.name, type=role.type))
+            return dict(roles=return_roles, page=page, page_limit=page_limit, total=total_roles)
+        else:
+            raise RequestParameterInvalidException("The value of 'scope' parameter is invalid. Alllowed values: current, available")
+
+    def set_permissions(self, trans, encoded_folder_id, payload: dict):
+        """
+        Set permissions of the given folder to the given role ids.
+
+        :param  encoded_folder_id:      the encoded id of the folder to set the permissions of
+        :type   encoded_folder_id:      an encoded id string
+        :param   payload: dictionary structure containing:
+
+            :param  action:            (required) describes what action should be performed
+            :type   action:            string
+            :param  add_ids[]:         list of Role.id defining roles that should have add item permission on the folder
+            :type   add_ids[]:         string or list
+            :param  manage_ids[]:      list of Role.id defining roles that should have manage permission on the folder
+            :type   manage_ids[]:      string or list
+            :param  modify_ids[]:      list of Role.id defining roles that should have modify permission on the folder
+            :type   modify_ids[]:      string or list
+
+        :type       dictionary
+        :returns:   dict of current roles for all available permission types.
+        :rtype:     dictionary
+        :raises: RequestParameterInvalidException, InsufficientPermissionsException, RequestParameterMissingException
+        """
+
+        is_admin = trans.user_is_admin
+        current_user_roles = trans.get_current_user_roles()
+        decoded_folder_id = self.folder_manager.cut_and_decode(trans, encoded_folder_id)
+        folder = self.folder_manager.get(trans, decoded_folder_id)
+        if not (is_admin or trans.app.security_agent.can_manage_library_item(current_user_roles, folder)):
+            raise InsufficientPermissionsException('You do not have proper permission to modify permissions of this folder.')
+
+        new_add_roles_ids = util.listify(payload.get('add_ids[]', None))
+        new_manage_roles_ids = util.listify(payload.get('manage_ids[]', None))
+        new_modify_roles_ids = util.listify(payload.get('modify_ids[]', None))
+
+        action = payload.get('action', None)
+        if action is None:
+            raise RequestParameterMissingException('The mandatory parameter "action" is missing.')
+        elif action == 'set_permissions':
+
+            # ADD TO LIBRARY ROLES
+            valid_add_roles = []
+            invalid_add_roles_names = []
+            for role_id in new_add_roles_ids:
+                role = self.role_manager.get(trans, self.__decode_id(trans, role_id, 'role'))
+                #  Check whether role is in the set of allowed roles
+                valid_roles, total_roles = trans.app.security_agent.get_valid_roles(trans, folder)
+                if role in valid_roles:
+                    valid_add_roles.append(role)
+                else:
+                    invalid_add_roles_names.append(role_id)
+            if len(invalid_add_roles_names) > 0:
+                log.warning("The following roles could not be added to the add library item permission: " + str(invalid_add_roles_names))
+
+            # MANAGE FOLDER ROLES
+            valid_manage_roles = []
+            invalid_manage_roles_names = []
+            for role_id in new_manage_roles_ids:
+                role = self.role_manager.get(trans, self.__decode_id(trans, role_id, 'role'))
+                #  Check whether role is in the set of allowed roles
+                valid_roles, total_roles = trans.app.security_agent.get_valid_roles(trans, folder)
+                if role in valid_roles:
+                    valid_manage_roles.append(role)
+                else:
+                    invalid_manage_roles_names.append(role_id)
+            if len(invalid_manage_roles_names) > 0:
+                log.warning("The following roles could not be added to the manage folder permission: " + str(invalid_manage_roles_names))
+
+            # MODIFY FOLDER ROLES
+            valid_modify_roles = []
+            invalid_modify_roles_names = []
+            for role_id in new_modify_roles_ids:
+                role = self.role_manager.get(trans, self.__decode_id(trans, role_id, 'role'))
+                #  Check whether role is in the set of allowed roles
+                valid_roles, total_roles = trans.app.security_agent.get_valid_roles(trans, folder)
+                if role in valid_roles:
+                    valid_modify_roles.append(role)
+                else:
+                    invalid_modify_roles_names.append(role_id)
+            if len(invalid_modify_roles_names) > 0:
+                log.warning("The following roles could not be added to the modify folder permission: " + str(invalid_modify_roles_names))
+
+            permissions = {trans.app.security_agent.permitted_actions.LIBRARY_ADD: valid_add_roles}
+            permissions.update({trans.app.security_agent.permitted_actions.LIBRARY_MANAGE: valid_manage_roles})
+            permissions.update({trans.app.security_agent.permitted_actions.LIBRARY_MODIFY: valid_modify_roles})
+
+            trans.app.security_agent.set_all_library_permissions(trans, folder, permissions)
+        else:
+            raise RequestParameterInvalidException('The mandatory parameter "action" has an invalid value.'
+                                                   'Allowed values are: "set_permissions"')
+        return self.folder_manager.get_current_roles(trans, folder)
+
+    def delete(self, trans, encoded_folder_id, undelete: bool = False):
+        """
+        DELETE /api/folders/{encoded_folder_id}
+
+        Mark the folder with the given ``encoded_folder_id`` as `deleted`
+        (or remove the `deleted` mark if the `undelete` param is true).
+
+        .. note:: Currently, only admin users can un/delete folders.
+
+        :param  encoded_folder_id:     the encoded id of the folder to un/delete
+        :type   encoded_folder_id:     an encoded id string
+
+        :param  undelete:    (optional) flag specifying whether the item should be deleted or undeleted, defaults to false:
+        :type   undelete:    bool
+
+        :returns:   detailed folder information
+        :rtype:     dictionary
+
+        """
+        folder = self.folder_manager.get(trans, self.folder_manager.cut_and_decode(trans, encoded_folder_id), True)
+        folder = self.folder_manager.delete(trans, folder, undelete)
+        folder_dict = self.folder_manager.get_folder_dict(trans, folder)
+        return folder_dict
+
+    def update(self, trans, encoded_folder_id, payload: dict):
+        """
+        Update the folder defined by an ``encoded_folder_id``
+        with the data in the payload.
+
+       .. note:: Currently, only admin users can update library folders. Also the folder must not be `deleted`.
+
+        :param  id:      the encoded id of the folder
+        :type   id:      an encoded id string
+
+        :param  payload: (required) dictionary structure containing::
+            'name':         new folder's name, cannot be empty
+            'description':  new folder's description
+        :type   payload: dict
+
+        :returns:   detailed folder information
+        :rtype:     dict
+
+        :raises: RequestParameterMissingException
+        """
+        decoded_folder_id = self.folder_manager.cut_and_decode(trans, encoded_folder_id)
+        folder = self.folder_manager.get(trans, decoded_folder_id)
+        name = payload.get('name', None)
+        if not name:
+            raise RequestParameterMissingException("Parameter 'name' of library folder is required. You cannot remove it.")
+        description = payload.get('description', None)
+        updated_folder = self.folder_manager.update(trans, folder, name, description)
+        folder_dict = self.folder_manager.get_folder_dict(trans, updated_folder)
+        return folder_dict
+
+    def __decode_id(self, trans, encoded_id, object_name=None):
+        """
+        Try to decode the id.
+
+        :param  object_name:      Name of the object the id belongs to. (optional)
+        :type   object_name:      str
+        """
+        try:
+            return trans.security.decode_id(encoded_id)
+        except TypeError:
+            raise MalformedId('Malformed %s id specified, unable to decode.' % object_name if object_name is not None else '')
+        except ValueError:
+            raise MalformedId('Wrong %s id specified, unable to decode.' % object_name if object_name is not None else '')

lib/galaxy/managers/group_roles.py

@@ -7,7 +7,7 @@ from typing import (
 )
 
 from galaxy import model
-from galaxy.app import StructuredApp
+from galaxy.app import MinimalManagerApp
 from galaxy.exceptions import (
     ObjectNotFound,
 )
@@ -22,7 +22,7 @@ log = logging.getLogger(__name__)
 class GroupRolesManager:
     """Interface/service object shared by controllers for interacting with group roles."""
 
-    def __init__(self, app: StructuredApp) -> None:
+    def __init__(self, app: MinimalManagerApp) -> None:
         self._app = app
 
     def index(self, trans: ProvidesAppContext, group_id: EncodedDatabaseIdField) -> List[Dict[str, Any]]:

lib/galaxy/managers/group_users.py

@@ -7,7 +7,7 @@ from typing import (
 )
 
 from galaxy import model
-from galaxy.app import StructuredApp
+from galaxy.app import MinimalManagerApp
 from galaxy.exceptions import (
     ObjectNotFound,
 )
@@ -22,7 +22,7 @@ log = logging.getLogger(__name__)
 class GroupUsersManager:
     """Interface/service object shared by controllers for interacting with group users."""
 
-    def __init__(self, app: StructuredApp) -> None:
+    def __init__(self, app: MinimalManagerApp) -> None:
         self._app = app
 
     def index(self, trans: ProvidesAppContext, group_id: EncodedDatabaseIdField) -> List[Dict[str, Any]]:

lib/galaxy/managers/groups.py

@@ -7,7 +7,7 @@ from typing import (
 from sqlalchemy import false
 
 from galaxy import model
-from galaxy.app import StructuredApp
+from galaxy.app import MinimalManagerApp
 from galaxy.exceptions import (
     Conflict,
     ObjectAttributeMissingException,
@@ -22,7 +22,7 @@ from galaxy.web import url_for
 class GroupsManager:
     """Interface/service object shared by controllers for interacting with groups."""
 
-    def __init__(self, app: StructuredApp) -> None:
+    def __init__(self, app: MinimalManagerApp) -> None:
         self._app = app
 
     def index(self, trans: ProvidesAppContext):

lib/galaxy/managers/hdas.py

@@ -20,7 +20,7 @@ from galaxy.managers import (
     taggable,
     users,
 )
-from galaxy.structured_app import StructuredApp
+from galaxy.structured_app import MinimalManagerApp
 
 log = logging.getLogger(__name__)
 
@@ -41,7 +41,7 @@ class HDAManager(datasets.DatasetAssociationManager,
     # TODO: move what makes sense into DatasetManager
     # TODO: which of these are common with LDDAs and can be pushed down into DatasetAssociationManager?
 
-    def __init__(self, app: StructuredApp, user_manager: users.UserManager):
+    def __init__(self, app: MinimalManagerApp, user_manager: users.UserManager):
         """
         Set up and initialize other managers needed by hdas.
         """
@@ -135,6 +135,13 @@ class HDAManager(datasets.DatasetAssociationManager,
 
     # .... deletion and purging
     def purge(self, hda, flush=True):
+        if self.app.config.enable_celery_tasks:
+            from galaxy.celery.tasks import purge_hda
+            purge_hda.delay(hda_id=hda.id)
+        else:
+            self._purge(hda, flush=flush)
+
+    def _purge(self, hda, flush=True):
         """
         Purge this HDA and the dataset underlying it.
         """
@@ -146,7 +153,6 @@ class HDAManager(datasets.DatasetAssociationManager,
         # decrease the user's space used
         if quota_amount_reduction:
             user.adjust_total_disk_usage(-quota_amount_reduction)
-        return hda
 
     # .... states
     def error_if_uploading(self, hda):
@@ -248,7 +254,7 @@ class HDASerializer(  # datasets._UnflattenedMetadataDatasetAssociationSerialize
         annotatable.AnnotatableSerializerMixin):
     model_manager_class = HDAManager
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         super().__init__(app)
         self.hda_manager = self.manager
 
@@ -507,7 +513,7 @@ class HDADeserializer(datasets.DatasetAssociationDeserializer,
     """
     model_manager_class = HDAManager
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         super().__init__(app)
         self.hda_manager = self.manager
 

lib/galaxy/managers/hdcas.py

@@ -16,7 +16,7 @@ from galaxy.managers import (
     taggable
 )
 from galaxy.managers.collections_util import get_hda_and_element_identifiers
-from galaxy.structured_app import StructuredApp
+from galaxy.structured_app import MinimalManagerApp
 from galaxy.util.zipstream import ZipstreamWrapper
 
 
@@ -87,7 +87,7 @@ class DCESerializer(base.ModelSerializer):
     Serializer for DatasetCollectionElements.
     """
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         super().__init__(app)
         self.hda_serializer = hdas.HDASerializer(app)
         self.dc_serializer = DCSerializer(app, dce_serializer=self)
@@ -121,7 +121,7 @@ class DCSerializer(base.ModelSerializer):
     Serializer for DatasetCollections.
     """
 
-    def __init__(self, app: StructuredApp, dce_serializer=None):
+    def __init__(self, app: MinimalManagerApp, dce_serializer=None):
         super().__init__(app)
         self.dce_serializer = dce_serializer or DCESerializer(app)
 
@@ -160,7 +160,7 @@ class DCASerializer(base.ModelSerializer):
     Base (abstract) Serializer class for HDCAs and LDCAs.
     """
 
-    def __init__(self, app: StructuredApp, dce_serializer=None):
+    def __init__(self, app: MinimalManagerApp, dce_serializer=None):
         super().__init__(app)
         self.dce_serializer = dce_serializer or DCESerializer(app)
 
@@ -215,7 +215,7 @@ class HDCASerializer(
     Serializer for HistoryDatasetCollectionAssociations.
     """
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         super().__init__(app)
         self.hdca_manager = HDCAManager(app)
 

lib/galaxy/managers/histories.py

@@ -5,6 +5,7 @@ Histories are containers for datasets or dataset collections
 created (or copied) by users over the course of an analysis.
 """
 import logging
+from typing import Optional
 
 from sqlalchemy import (
     asc,
@@ -21,7 +22,8 @@ from galaxy.managers import (
     history_contents,
     sharable
 )
-from galaxy.structured_app import StructuredApp
+from galaxy.schema.fields import EncodedDatabaseIdField
+from galaxy.structured_app import MinimalManagerApp
 
 log = logging.getLogger(__name__)
 
@@ -38,7 +40,7 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix
 
     # TODO: incorporate imp/exp (or alias to)
 
-    def __init__(self, app: StructuredApp, hda_manager: hdas.HDAManager, contents_manager: history_contents.HistoryContentsManager, contents_filters: history_contents.HistoryContentsFilters):
+    def __init__(self, app: MinimalManagerApp, hda_manager: hdas.HDAManager, contents_manager: history_contents.HistoryContentsManager, contents_filters: history_contents.HistoryContentsFilters):
         super().__init__(app)
         self.hda_manager = hda_manager
         self.contents_manager = contents_manager
@@ -172,7 +174,7 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix
 
 class HistoryExportView:
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         self.app = app
 
     def get_exports(self, trans, history_id):
@@ -224,7 +226,7 @@ class HistorySerializer(sharable.SharableModelSerializer, deletable.PurgableSeri
     model_manager_class = HistoryManager
     SINGLE_CHAR_ABBR = 'h'
 
-    def __init__(self, app: StructuredApp, hda_manager: hdas.HDAManager, hda_serializer: hdas.HDASerializer, history_contents_serializer: history_contents.HistoryContentsSerializer):
+    def __init__(self, app: MinimalManagerApp, hda_manager: hdas.HDAManager, hda_serializer: hdas.HDASerializer, history_contents_serializer: history_contents.HistoryContentsSerializer):
         super().__init__(app)
 
         self.history_manager = self.manager
@@ -443,7 +445,7 @@ class HistoryDeserializer(sharable.SharableModelDeserializer, deletable.Purgable
     """
     model_manager_class = HistoryManager
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         super().__init__(app)
         self.history_manager = self.manager
 
@@ -471,3 +473,25 @@ class HistoryFilters(sharable.SharableModelFilters, deletable.PurgableFiltersMix
             'create_time': {'op': ('le', 'ge', 'gt', 'lt'), 'val': self.parse_date},
             'update_time': {'op': ('le', 'ge', 'gt', 'lt'), 'val': self.parse_date},
         })
+
+
+class HistoriesService:
+    """Common interface/service logic for interactions with histories in the context of the API.
+
+    Provides the logic of the actions invoked by API controllers and uses type definitions
+    and pydantic models to declare its parameters and return types.
+    """
+
+    def __init__(self, app: MinimalManagerApp, manager: HistoryManager, serializer: HistorySerializer):
+        self.app = app
+        self.manager = manager
+        self.serializer = serializer
+        self.shareable_service = sharable.ShareableService(self.manager, self.serializer)
+
+    # TODO: add the rest of the API actions here and call them directly from the API controller
+
+    def sharing(self, trans, id: EncodedDatabaseIdField, payload: Optional[sharable.SharingPayload] = None) -> sharable.SharingStatus:
+        """Allows to publish or share with other users the given resource (by id) and returns the current sharing
+        status of the resource.
+        """
+        return self.shareable_service.sharing(trans, id, payload)

lib/galaxy/managers/history_contents.py

@@ -32,7 +32,7 @@ from galaxy.managers import (
     taggable,
     tools
 )
-from galaxy.structured_app import StructuredApp
+from galaxy.structured_app import MinimalManagerApp
 
 log = logging.getLogger(__name__)
 
@@ -73,7 +73,7 @@ class HistoryContentsManager(containers.ContainerManagerMixin):
     )
     default_order_by = 'hid'
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         self.app = app
         self.contained_manager = app[self.contained_class_manager_class]
         self.subcontainer_manager = app[self.subcontainer_class_manager_class]
@@ -427,7 +427,7 @@ class HistoryContentsSerializer(base.ModelSerializer, deletable.PurgableSerializ
     """
     model_manager_class = HistoryContentsManager
 
-    def __init__(self, app: StructuredApp, **kwargs):
+    def __init__(self, app: MinimalManagerApp, **kwargs):
         super().__init__(app, **kwargs)
 
         self.default_view = 'summary'

lib/galaxy/managers/jobs.py

@@ -9,6 +9,7 @@ from pydantic import (
 )
 from sqlalchemy import and_, false, func, or_
 from sqlalchemy.orm import aliased
+from sqlalchemy.orm.scoping import scoped_session
 from sqlalchemy.sql import select
 
 from galaxy import model
@@ -21,6 +22,7 @@ from galaxy.managers.collections import DatasetCollectionManager
 from galaxy.managers.datasets import DatasetManager
 from galaxy.managers.hdas import HDAManager
 from galaxy.managers.lddas import LDDAManager
+from galaxy.security.idencoding import IdEncodingHelper
 from galaxy.structured_app import StructuredApp
 from galaxy.util import (
     defaultdict,
@@ -96,17 +98,17 @@ class JobSearch:
     """Search for jobs using tool inputs or other jobs"""
     def __init__(
         self,
-        app: StructuredApp,
+        sa_session: scoped_session,
         hda_manager: HDAManager,
         dataset_collection_manager: DatasetCollectionManager,
-        ldda_manager: LDDAManager
+        ldda_manager: LDDAManager,
+        id_encoding_helper: IdEncodingHelper,
     ):
-        self.app = app
-        self.sa_session = app.model.context
+        self.sa_session = sa_session
         self.hda_manager = hda_manager
         self.dataset_collection_manager = dataset_collection_manager
         self.ldda_manager = ldda_manager
-        self.decode_id = self.app.security.decode_id
+        self.decode_id = id_encoding_helper.decode_id
 
     def by_tool_input(self, trans, tool_id, tool_version, param=None, param_dump=None, job_state='ok'):
         """Search for jobs producing same results using the 'inputs' part of a tool POST."""
@@ -394,6 +396,8 @@ class JobSearch:
 def view_show_job(trans, job, full: bool) -> typing.Dict:
     is_admin = trans.user_is_admin
     job_dict = trans.app.security.encode_all_ids(job.to_dict('element', system_details=is_admin), True)
+    if trans.app.config.expose_dataset_path and 'command_line' not in job_dict:
+        job_dict['command_line'] = job.command_line
     if full:
         job_dict.update(dict(
             tool_stdout=job.tool_stdout,

lib/galaxy/managers/lddas.py

@@ -3,7 +3,7 @@ import logging
 from galaxy import model, util
 from galaxy.managers import base as manager_base
 from galaxy.managers.datasets import DatasetAssociationManager
-from galaxy.structured_app import StructuredApp
+from galaxy.structured_app import MinimalManagerApp
 
 log = logging.getLogger(__name__)
 
@@ -14,7 +14,7 @@ class LDDAManager(DatasetAssociationManager):
     """
     model_class = model.LibraryDatasetDatasetAssociation
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         """
         Set up and initialize other managers needed by lddas.
         """

lib/galaxy/managers/metrics.py

@@ -13,7 +13,7 @@ from pydantic import (
     Field,
 )
 
-from galaxy.app import StructuredApp
+from galaxy.app import MinimalManagerApp
 
 log = logging.getLogger(__name__)
 
@@ -72,7 +72,7 @@ TimeSeriesTupleGenerator = Generator[TimeSeriesTuple, None, None]
 class MetricsManager:
     """Interface/service object shared by controllers for interacting with metrics."""
 
-    def __init__(self, app: StructuredApp) -> None:
+    def __init__(self, app: MinimalManagerApp) -> None:
         self._app = app
         #: set to true to send additional debugging info to the log
         self.debugging = True

lib/galaxy/managers/pages.py

@@ -32,7 +32,7 @@ from galaxy.managers.markdown_util import (
 )
 from galaxy.model.item_attrs import UsesAnnotations
 from galaxy.schema.fields import EncodedDatabaseIdField
-from galaxy.structured_app import StructuredApp
+from galaxy.structured_app import MinimalManagerApp
 from galaxy.util import unicodify
 from galaxy.util.sanitize_html import sanitize_html
 
@@ -190,13 +190,17 @@ class PageSummaryList(BaseModel):
     )
 
 
-class PagesManager:
-    """
-    Common interface/service logic for interactions with pages.
+class PagesService:
+    """Common interface/service logic for interactions with pages in the context of the API.
+
+    Provides the logic of the actions invoked by API controllers and uses type definitions
+    and pydantic models to declare its parameters and return types.
     """
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         self.manager = PageManager(app)
+        self.serializer = PageSerializer(app)
+        self.shareable_service = sharable.ShareableService(self.manager, self.serializer)
 
     def index(self, trans, deleted: bool = False) -> PageSummaryList:
         """Return a list of Pages viewable by the user
@@ -252,8 +256,7 @@ class PagesManager:
         trans.sa_session.flush()
 
     def show(self, trans, id: EncodedDatabaseIdField) -> PageDetails:
-        """
-            View a page summary and the content of the latest revision
+        """View a page summary and the content of the latest revision
 
         :param  id:    ID of page to be displayed
 
@@ -269,8 +272,6 @@ class PagesManager:
 
     def show_pdf(self, trans, id: EncodedDatabaseIdField):
         """
-        GET /api/pages/{id}.pdf
-
         View a page summary and the content of the latest revision as PDF.
 
         :param  id: ID of page to be displayed
@@ -285,10 +286,15 @@ class PagesManager:
         trans.response.set_content_type("application/pdf")
         return internal_galaxy_markdown_to_pdf(trans, internal_galaxy_markdown, 'page')
 
+    def sharing(self, trans, id: EncodedDatabaseIdField, payload: Optional[sharable.SharingPayload] = None) -> sharable.SharingStatus:
+        """Allows to publish or share with other users the given resource (by id) and returns the current sharing
+        status of the resource.
+        """
+        return self.shareable_service.sharing(trans, id, payload)
+
 
 class PageManager(sharable.SharableModelManager, UsesAnnotations):
-    """
-    """
+    """Provides operations for managing a Page."""
 
     model_class = model.Page
     foreign_key_name = 'page'
@@ -298,7 +304,7 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations):
     annotation_assoc = model.PageAnnotationAssociation
     rating_assoc = model.PageRatingAssociation
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         """
         """
         super().__init__(app)
@@ -421,7 +427,7 @@ class PageSerializer(sharable.SharableModelSerializer):
     model_manager_class = PageManager
     SINGLE_CHAR_ABBR = 'p'
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         super().__init__(app)
         self.page_manager = PageManager(app)
 
@@ -442,7 +448,7 @@ class PageDeserializer(sharable.SharableModelDeserializer):
     """
     model_manager_class = PageManager
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         super().__init__(app)
         self.page_manager = self.manager
 

lib/galaxy/managers/remote_files.py

@@ -12,7 +12,7 @@ from typing import (
 from pydantic.tools import parse_obj_as
 
 from galaxy import exceptions
-from galaxy.app import StructuredApp
+from galaxy.app import MinimalManagerApp
 from galaxy.files import (
     ConfiguredFileSources,
     ProvidesUserFileSourcesUserContext,
@@ -37,7 +37,7 @@ class RemoteFilesManager:
     Interface/service object for interacting with remote files.
     """
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         self._app = app
 
     def index(

lib/galaxy/managers/sharable.py

@@ -11,8 +11,16 @@ A sharable Galaxy object:
 """
 import logging
 import re
-from typing import Optional, Type
+from typing import (
+    List,
+    Optional,
+    Type,
+)
 
+from pydantic import (
+    BaseModel,
+    Field,
+)
 from sqlalchemy import true
 
 from galaxy import exceptions
@@ -25,7 +33,9 @@ from galaxy.managers import (
     users
 )
 from galaxy.model import UserShareAssociation
-from galaxy.structured_app import StructuredApp
+from galaxy.schema.fields import EncodedDatabaseIdField
+from galaxy.structured_app import MinimalManagerApp
+from galaxy.util import ready_name_for_url
 
 log = logging.getLogger(__name__)
 
@@ -41,7 +51,7 @@ class SharableModelManager(base.ModelManager, secured.OwnableManagerMixin, secur
     #: the single character abbreviation used in username_and_slug: e.g. 'h' for histories: u/user/h/slug
     SINGLE_CHAR_ABBR: Optional[str] = None
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         super().__init__(app)
         # user manager is needed to check access/ownership/admin
         self.user_manager = users.UserManager(app)
@@ -461,3 +471,222 @@ class SharableModelFilters(base.ModelFilterParser,
             # chose by user should prob. only be available for admin? (most often we'll only need trans.user)
             # 'user'          : { 'op': ( 'eq' ), 'val': self.parse_id_list },
         })
+
+
+class SharingPayload(BaseModel):
+    action: str = Field(  # TODO: this seems like it should be a list of actions instead of separating them by '-'
+        ...,  # Mark this field as required
+        title="Action",
+        description=(
+            "The name of the sharing action. "
+            "Can be one (or multiple values separated by '-') of the following: "
+            "make_accessible_via_link, make_accessible_and_publish, publish, "
+            "unpublish, disable_link_access, disable_link_access_and_unpublish, unshare_user"
+        ),
+    )
+    user_id: Optional[EncodedDatabaseIdField] = Field(
+        None,
+        title="User ID",
+        description=(
+            "The ID of the user with whom this resource will be shared. "
+            "*Required* when the action is `unshare_user`."
+        ),
+    )
+
+
+class UserEmail(BaseModel):
+    id: EncodedDatabaseIdField = Field(
+        ...,  # Mark this field as required
+        title="User ID",
+        description="The encoded ID of the user.",
+    )
+    email: str = Field(
+        ...,  # Mark this field as required
+        title="Email",
+        description="The email of the user.",
+    )
+
+
+class SharingStatus(BaseModel):
+    id: EncodedDatabaseIdField = Field(
+        ...,  # Mark this field as required
+        title="ID",
+        description="The encoded ID of the resource to be shared.",
+    )
+    title: str = Field(
+        ...,  # Mark this field as required
+        title="Title",
+        description="The title or name of the resource.",
+    )
+    importable: bool = Field(
+        ...,  # Mark this field as required
+        title="Importable",
+        description="Whether this resource can be published using a link.",
+    )
+    published: bool = Field(
+        ...,  # Mark this field as required
+        title="Published",
+        description="Whether this resource is currently published.",
+    )
+    users_shared_with: List[UserEmail] = Field(
+        [],
+        title="Users shared with",
+        description="The list of encoded ids for users the resource has been shared.",
+    )
+    username_and_slug: Optional[str] = Field(
+        None,
+        title="Username and slug",
+        description="The relative URL in the form of /u/{username}/{resource_single_char}/{slug}",
+    )
+    skipped: Optional[bool] = Field(
+        None,
+        title="Skipped",
+        description="Indicates that some of the resources within this object were not published due to an error.",
+    )
+
+
+class SlugBuilder:
+    """Builder for creating slugs out of items."""
+
+    def create_item_slug(self, sa_session, item) -> bool:
+        """Create/set item slug.
+
+        Slug is unique among user's importable items for item's class.
+
+        :param sa_session: Database session context.
+        :param item: The item to create/update its slug.
+        :type item: [type]
+        :return: Returns true if item's slug was set/changed; false otherwise.
+        :rtype: bool
+        """
+        cur_slug = item.slug
+
+        # Setup slug base.
+        if cur_slug is None or cur_slug == '':
+            # Item can have either a name or a title.
+            item_name = ''
+            if hasattr(item, 'name'):
+                item_name = item.name
+            elif hasattr(item, 'title'):
+                item_name = item.title
+            slug_base = ready_name_for_url(item_name.lower())
+        else:
+            slug_base = cur_slug
+
+        # Using slug base, find a slug that is not taken. If slug is taken,
+        # add integer to end.
+        new_slug = slug_base
+        count = 1
+        # Ensure unique across model class and user and don't include this item
+        # in the check in case it has previously been assigned a valid slug.
+        while sa_session.query(item.__class__).filter(item.__class__.user == item.user, item.__class__.slug == new_slug, item.__class__.id != item.id).count() != 0:
+            # Slug taken; choose a new slug based on count. This approach can
+            # handle numerous items with the same name gracefully.
+            new_slug = f'{slug_base}-{count}'
+            count += 1
+
+        # Set slug and return.
+        item.slug = new_slug
+        return item.slug == cur_slug
+
+
+class ShareableService:
+    """ Provides the logic used by the API to share resources with other users."""
+
+    def __init__(self, manager: SharableModelManager, serializer: SharableModelSerializer) -> None:
+        self.manager = manager
+        self.serializer = serializer
+        self.slug_builder = SlugBuilder()
+
+    def sharing(self, trans, id: EncodedDatabaseIdField, payload: Optional[SharingPayload] = None) -> SharingStatus:
+        """Allows to publish or share with other users the given resource (by id) and returns the current sharing
+        status of the resource.
+
+        :param id: The encoded ID of the resource to share.
+        :type id: EncodedDatabaseIdField
+        :param payload: The options to share this resource, defaults to None
+        :type payload: Optional[sharable.SharingPayload], optional
+        :return: The current sharing status of the resource.
+        :rtype: sharable.SharingStatus
+        """
+        skipped = False
+        class_name = self.manager.model_class.__name__
+        item = base.get_object(trans, id, class_name, check_ownership=True, check_accessible=True, deleted=False)
+        actions = []
+        if payload:
+            actions += payload.action.split("-")
+        for action in actions:
+            if action == "make_accessible_via_link":
+                self._make_item_accessible(trans.sa_session, item)
+                if hasattr(item, "has_possible_members") and item.has_possible_members:
+                    skipped = self._make_members_public(trans, item)
+            elif action == "make_accessible_and_publish":
+                self._make_item_accessible(trans.sa_session, item)
+                if hasattr(item, "has_possible_members") and item.has_possible_members:
+                    skipped = self._make_members_public(trans, item)
+                item.published = True
+            elif action == "publish":
+                if item.importable:
+                    item.published = True
+                    if hasattr(item, "has_possible_members") and item.has_possible_members:
+                        skipped = self._make_members_public(trans, item)
+                else:
+                    raise exceptions.MessageException("%s not importable." % class_name)
+            elif action == "disable_link_access":
+                item.importable = False
+            elif action == "unpublish":
+                item.published = False
+            elif action == "disable_link_access_and_unpublish":
+                item.importable = item.published = False
+            elif action == "unshare_user":
+                if payload is None or payload.user_id is None:
+                    raise exceptions.MessageException(f"Missing required user_id to perform {action}")
+                user = trans.sa_session.query(trans.app.model.User).get(trans.app.security.decode_id(payload.user_id))
+                class_name_lc = class_name.lower()
+                ShareAssociation = getattr(trans.app.model, "%sUserShareAssociation" % class_name)
+                usas = trans.sa_session.query(ShareAssociation).filter_by(**{"user": user, class_name_lc: item}).all()
+                if not usas:
+                    raise exceptions.MessageException("%s was not shared with user." % class_name)
+                for usa in usas:
+                    trans.sa_session.delete(usa)
+            trans.sa_session.add(item)
+            trans.sa_session.flush()
+        if item.importable and not item.slug:
+            self._make_item_accessible(trans.sa_session, item)
+        item_dict = self.serializer.serialize_to_view(item,
+            user=trans.user, trans=trans, default_view="sharing")
+        item_dict["users_shared_with"] = [{"id": self.manager.app.security.encode_id(a.user.id), "email": a.user.email} for a in item.users_shared_with]
+        if skipped:
+            item_dict["skipped"] = True
+        return SharingStatus.parse_obj(item_dict)
+
+    def _is_valid_slug(self, slug):
+        """ Returns true if slug is valid. """
+        return base.is_valid_slug(slug)
+
+    def _make_item_accessible(self, sa_session, item):
+        """ Makes item accessible--viewable and importable--and sets item's slug.
+            Does not flush/commit changes, however. Item must have name, user,
+            importable, and slug attributes. """
+        item.importable = True
+        self.slug_builder.create_item_slug(sa_session, item)
+
+    def _make_members_public(self, trans, item):
+        """ Make the non-purged datasets in history public
+        Performs pemissions check.
+        """
+        # TODO eventually we should handle more classes than just History
+        skipped = False
+        for hda in item.activatable_datasets:
+            dataset = hda.dataset
+            if not trans.app.security_agent.dataset_is_public(dataset):
+                if trans.app.security_agent.can_manage_dataset(trans.user.all_roles(), dataset):
+                    try:
+                        trans.app.security_agent.make_dataset_public(hda.dataset)
+                    except Exception:
+                        log.warning("Unable to make dataset with id: %s public", dataset.id)
+                        skipped = True
+                else:
+                    log.warning("User without permissions tried to make dataset with id: %s public", dataset.id)
+                    skipped = True
+        return skipped

lib/galaxy/managers/users.py

@@ -30,7 +30,7 @@ from galaxy.security.validate_user_input import (
     validate_password,
     validate_publicname
 )
-from galaxy.structured_app import BasicApp, StructuredApp
+from galaxy.structured_app import BasicApp, MinimalManagerApp
 from galaxy.util.hash_util import new_secure_hash
 from galaxy.web import url_for
 
@@ -611,7 +611,7 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin):
 class UserSerializer(base.ModelSerializer, deletable.PurgableSerializerMixin):
     model_manager_class = UserManager
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         """
         Convert a User and associated data to a dictionary representation.
         """

lib/galaxy/managers/visualizations.py

@@ -5,10 +5,12 @@ Visualizations are saved configurations/variables used to
 reproduce a specific view in a Galaxy visualization.
 """
 import logging
+from typing import Optional
 
 from galaxy import model
 from galaxy.managers import sharable
-from galaxy.structured_app import StructuredApp
+from galaxy.schema.fields import EncodedDatabaseIdField
+from galaxy.structured_app import MinimalManagerApp
 
 log = logging.getLogger(__name__)
 
@@ -41,7 +43,7 @@ class VisualizationSerializer(sharable.SharableModelSerializer):
     model_manager_class = VisualizationManager
     SINGLE_CHAR_ABBR = 'v'
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         super().__init__(app)
         self.visualization_manager = self.manager
 
@@ -71,3 +73,25 @@ class VisualizationDeserializer(sharable.SharableModelDeserializer):
         self.deserializers.update({
         })
         self.deserializable_keyset.update(self.deserializers.keys())
+
+
+class VisualizationsService:
+    """Common interface/service logic for interactions with visualizations in the context of the API.
+
+    Provides the logic of the actions invoked by API controllers and uses type definitions
+    and pydantic models to declare its parameters and return types.
+    """
+
+    def __init__(self, app: MinimalManagerApp, manager: VisualizationManager, serializer: VisualizationSerializer):
+        self.app = app
+        self.manager = manager
+        self.serializer = serializer
+        self.shareable_service = sharable.ShareableService(self.manager, self.serializer)
+
+    # TODO: add the rest of the API actions here and call them directly from the API controller
+
+    def sharing(self, trans, id: EncodedDatabaseIdField, payload: Optional[sharable.SharingPayload] = None) -> sharable.SharingStatus:
+        """Allows to publish or share with other users the given resource (by id) and returns the current sharing
+        status of the resource.
+        """
+        return self.shareable_service.sharing(trans, id, payload)

lib/galaxy/managers/workflows.py

@@ -26,7 +26,7 @@ from galaxy import (
 )
 from galaxy.jobs.actions.post import ActionBox
 from galaxy.model.item_attrs import UsesAnnotations
-from galaxy.structured_app import StructuredApp
+from galaxy.structured_app import MinimalManagerApp
 from galaxy.tools.parameters import (
     params_to_incoming,
     visit_input_values
@@ -69,7 +69,7 @@ class WorkflowsManager:
     the galaxy.workflow module.
     """
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         self.app = app
 
     def get_stored_workflow(self, trans, workflow_id, by_stored_id=True):
@@ -306,7 +306,7 @@ CreatedWorkflow = namedtuple("CreatedWorkflow", ["stored_workflow", "workflow",
 
 class WorkflowContentsManager(UsesAnnotations):
 
-    def __init__(self, app: StructuredApp):
+    def __init__(self, app: MinimalManagerApp):
         self.app = app
         self._resource_mapper_function = get_resource_mapper_function(app)
 
@@ -361,8 +361,9 @@ class WorkflowContentsManager(UsesAnnotations):
             raise exceptions.RequestParameterInvalidException(f"Invalid workflow format detected [{data}]")
 
         workflow_input_name = data['name']
-        if source:
-            name = f"{workflow_input_name} (imported from {source})"
+        imported_sufix = f"(imported from {source})"
+        if source and imported_sufix not in workflow_input_name:
+            name = f"{workflow_input_name} {imported_sufix}"
         else:
             name = workflow_input_name
         workflow, missing_tool_tups = self._workflow_from_raw_description(

lib/galaxy/model/__init__.py

@@ -401,7 +401,7 @@ class User(Dictifiable, RepresentById):
         extra_user_preferences = self.preferences.get('extra_user_preferences')
         if extra_user_preferences:
             try:
-                data = json.loads(extra_user_preferences)
+                data.update(json.loads(extra_user_preferences))
             except Exception:
                 pass
         return data
@@ -755,8 +755,11 @@ class Job(JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById):
         self.state_history = []
         self.imported = False
         self.handler = None
+        self.create_time = None
         self.exit_code = None
+        self.history_id = None
         self.job_messages = None
+        self.update_time = None
         self._init_metrics()
         self.state_history.append(JobStateHistory(self))
 
@@ -1538,7 +1541,7 @@ class ImplicitCollectionJobs(RepresentById):
 class ImplicitCollectionJobsJobAssociation(RepresentById):
 
     def __init__(self):
-        pass
+        self.implicit_collection_jobs_id = None
 
 
 class PostJobAction(RepresentById):
@@ -2694,10 +2697,11 @@ class DatasetInstance:
         INVALID = 'invalid'
         OK = 'ok'
 
-    def __init__(self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, tool_version=None, extension=None,
-                 dbkey=None, metadata=None, history=None, dataset=None, deleted=False, designation=None,
-                 parent_id=None, validated_state='unknown', validated_state_message=None, visible=True, create_dataset=False, sa_session=None,
-                 extended_metadata=None, flush=True):
+    def __init__(self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, tool_version=None,
+                 extension=None, dbkey=None, metadata=None, history=None, dataset=None, deleted=False,
+                 designation=None, parent_id=None, validated_state='unknown', validated_state_message=None,
+                 visible=True, create_dataset=False, sa_session=None, extended_metadata=None, flush=True,
+                 creating_job_id=None):
         self.name = name or "Unnamed dataset"
         self.id = id
         self.info = info
@@ -2720,6 +2724,7 @@ class DatasetInstance:
         if not dataset and create_dataset:
             # Had to pass the sqlalchemy session in order to create a new dataset
             dataset = Dataset(state=Dataset.states.NEW)
+            dataset.job_id = creating_job_id
             if flush:
                 sa_session.add(dataset)
                 sa_session.flush()
@@ -3085,6 +3090,7 @@ class DatasetInstance:
 
     @property
     def creating_job(self):
+        # TODO this should work with `return self.dataset.job` (revise failing unit tests)
         creating_job_associations = None
         if self.creating_job_associations:
             creating_job_associations = self.creating_job_associations
@@ -3310,6 +3316,8 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot
 
     def copy_tags_to(self, copy_tags=None):
         if copy_tags is not None:
+            if isinstance(copy_tags, dict):
+                copy_tags = copy_tags.values()
             for tag in copy_tags:
                 copied_tag = tag.copy(cls=HistoryDatasetAssociationTagAssociation)
                 self.tags.append(copied_tag)
@@ -3497,7 +3505,7 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot
                     continue
                 val = val.file_name
             # If no value for metadata, look in datatype for metadata.
-            elif val is None and hasattr(hda.datatype, name):
+            elif not hda.metadata.element_is_set(name) and hasattr(hda.datatype, name):
                 val = getattr(hda.datatype, name)
             rval['metadata_' + name] = val
         return rval
@@ -4286,7 +4294,7 @@ class DatasetCollection(Dictifiable, UsesAnnotations, RepresentById):
         error_message = f"Dataset collection has no {get_by_attribute} with key {key}."
         raise KeyError(error_message)
 
-    def copy(self, destination=None, element_destination=None, flush=True):
+    def copy(self, destination=None, element_destination=None, dataset_instance_attributes=None, flush=True):
         new_collection = DatasetCollection(
             collection_type=self.collection_type,
             element_count=self.element_count
@@ -4296,6 +4304,7 @@ class DatasetCollection(Dictifiable, UsesAnnotations, RepresentById):
                 new_collection,
                 destination=destination,
                 element_destination=element_destination,
+                dataset_instance_attributes=dataset_instance_attributes,
                 flush=flush
             )
         object_session(self).add(new_collection)
@@ -4537,7 +4546,7 @@ class HistoryDatasetCollectionAssociation(DatasetCollectionInstance,
                 break
         return matching_collection
 
-    def copy(self, element_destination=None):
+    def copy(self, element_destination=None, dataset_instance_attributes=None):
         """
         Create a copy of this history dataset collection association. Copy
         underlying collection.
@@ -4558,6 +4567,7 @@ class HistoryDatasetCollectionAssociation(DatasetCollectionInstance,
         collection_copy = self.collection.copy(
             destination=hdca,
             element_destination=element_destination,
+            dataset_instance_attributes=dataset_instance_attributes,
             flush=False,
         )
         hdca.collection = collection_copy
@@ -4719,17 +4729,22 @@ class DatasetCollectionElement(Dictifiable, RepresentById):
         else:
             return [element_object]
 
-    def copy_to_collection(self, collection, destination=None, element_destination=None, flush=True):
+    def copy_to_collection(self, collection, destination=None, element_destination=None, dataset_instance_attributes=None, flush=True):
+        dataset_instance_attributes = dataset_instance_attributes or {}
         element_object = self.element_object
         if element_destination:
             if self.is_collection:
                 element_object = element_object.copy(
                     destination=destination,
                     element_destination=element_destination,
+                    dataset_instance_attributes=dataset_instance_attributes,
                     flush=flush
                 )
             else:
                 new_element_object = element_object.copy(flush=flush, copy_tags=element_object.tags)
+                for attribute, value in dataset_instance_attributes.items():
+                    setattr(new_element_object, attribute, value)
+
                 new_element_object.visible = False
                 if destination is not None and element_object.hidden_beneath_collection_instance:
                     new_element_object.hidden_beneath_collection_instance = destination
@@ -4895,6 +4910,7 @@ class Workflow(Dictifiable, RepresentById):
         self.has_cycles = None
         self.has_errors = None
         self.steps = []
+        self.stored_workflow_id = None
         if uuid is None:
             self.uuid = uuid4()
         else:
@@ -4992,6 +5008,8 @@ class Workflow(Dictifiable, RepresentById):
         copied_workflow.has_cycles = self.has_cycles
         copied_workflow.has_errors = self.has_errors
         copied_workflow.reports_config = self.reports_config
+        copied_workflow.license = self.license
+        copied_workflow.creator_metadata = self.creator_metadata
 
         # Map old step ids to new steps
         step_mapping = {}
@@ -5321,6 +5339,7 @@ class WorkflowInvocation(UsesCreateAndUpdateTime, Dictifiable, RepresentById):
         self.subworkflow_invocations = []
         self.step_states = []
         self.steps = []
+        self.workflow_id = None
 
     def create_subworkflow_invocation_for_step(self, step):
         assert step.type == "subworkflow"
@@ -5673,6 +5692,11 @@ class WorkflowInvocationStep(Dictifiable, RepresentById):
         # CANCELLED = 'cancelled',  TODO: implement and expose
         # FAILED = 'failed',  TODO: implement and expose
 
+    def __init__(self):
+        self.implicit_collection_jobs_id = None
+        self.job_id = None
+        self.workflow_invocation_id = None
+
     @property
     def is_new(self):
         return self.state == self.states.NEW

lib/galaxy/model/database_utils.py

@@ -0,0 +1,125 @@
+import sqlite3
+from contextlib import contextmanager
+
+from sqlalchemy import create_engine
+from sqlalchemy.engine.url import make_url
+from sqlalchemy.sql.compiler import IdentifierPreparer
+from sqlalchemy.sql.expression import text
+
+from galaxy.exceptions import ConfigurationError
+
+
+def database_exists(db_url, database=None):
+    """Check if database exists; connect with db_url.
+
+    If database is None, use the database name from db_url.
+    """
+    dbm = DatabaseManager.make_manager(db_url, database)
+    return dbm.exists()
+
+
+def create_database(db_url, database=None, encoding='utf8', template=None):
+    """Create database; connect with db_url.
+
+    If database is None, use the database name from db_url.
+    """
+    dbm = DatabaseManager.make_manager(db_url, database)
+    dbm.create(encoding, template)
+
+
+@contextmanager
+def sqlalchemy_engine(url):
+    engine = create_engine(url)
+    try:
+        yield engine
+    finally:
+        engine.dispose()
+
+
+class DatabaseManager:
+
+    @staticmethod
+    def make_manager(db_url, database):
+        if db_url.startswith('postgres'):
+            return PosgresDatabaseManager(db_url, database)
+        elif db_url.startswith('sqlite'):
+            return SqliteDatabaseManager(db_url, database)
+        elif db_url.startswith('mysql'):
+            return MySQLDatabaseManager(db_url, database)
+        else:
+            raise ConfigurationError(f'Invalid database URL: {db_url}')
+
+    def __init__(self, db_url, database):
+        self.url = make_url(db_url)
+        self.database = database
+        if not database:
+            self._handle_no_database()
+
+
+class PosgresDatabaseManager(DatabaseManager):
+
+    def _handle_no_database(self):
+        self.database = self.url.database  # use database from db_url
+        self.url.database = 'postgres'
+
+    def exists(self):
+        with sqlalchemy_engine(self.url) as engine:
+            stmt = text('SELECT 1 FROM pg_database WHERE datname=:database')
+            stmt = stmt.bindparams(database=self.database)
+            with engine.connect() as conn:
+                return bool(conn.scalar(stmt))
+
+    def create(self, encoding, template):
+        with sqlalchemy_engine(self.url) as engine:
+            preparer = IdentifierPreparer(engine.dialect)
+            template = template or 'template1'
+            database, template = preparer.quote(self.database), preparer.quote(template)
+            stmt = f"CREATE DATABASE {database} ENCODING '{encoding}' TEMPLATE {template}"
+            with engine.connect().execution_options(isolation_level='AUTOCOMMIT') as conn:
+                conn.execute(stmt)
+
+
+class SqliteDatabaseManager(DatabaseManager):
+
+    def _handle_no_database(self):
+        self.database = self.url.database  # use database from db_url
+
+    def exists(self):
+
+        def can_connect_to_dbfile():
+            try:
+                sqlite3.connect(f'file:{db}?mode=ro', uri=True)
+            except sqlite3.OperationalError:
+                return False
+            else:
+                return True
+
+        db = self.url.database
+        # No database or ':memory:' creates an in-memory database
+        return not db or db == ':memory:' or can_connect_to_dbfile()
+
+    def create(self, *args):
+        # Ignore any args (encoding, template)
+        sqlite3.connect(f'file:{self.url.database}', uri=True)
+
+
+class MySQLDatabaseManager(DatabaseManager):
+
+    def _handle_no_database(self):
+        self.database = self.url.database  # use database from db_url
+
+    def exists(self):
+        with sqlalchemy_engine(self.url) as engine:
+            stmt = text("SELECT schema_name FROM information_schema.schemata WHERE schema_name=:database")
+            stmt = stmt.bindparams(database=self.database)
+            with engine.connect() as conn:
+                return bool(conn.scalar(stmt))
+
+    def create(self, encoding, *arg):
+        # Ignore any args (template)
+        with sqlalchemy_engine(self.url) as engine:
+            preparer = IdentifierPreparer(engine.dialect)
+            database = preparer.quote(self.database)
+            stmt = f"CREATE DATABASE {database} CHARACTER SET = '{encoding}'"
+            with engine.connect().execution_options(isolation_level='AUTOCOMMIT') as conn:
+                conn.execute(stmt)

lib/galaxy/model/dataset_collections/builder.py

@@ -1,27 +1,33 @@
 from galaxy import model
+from galaxy.util.oset import OrderedSet
 from .type_description import COLLECTION_TYPE_DESCRIPTION_FACTORY
 
 
-def build_collection(type, dataset_instances):
+def build_collection(type, dataset_instances, collection=None, associated_identifiers=None):
     """
     Build DatasetCollection with populated DatasetcollectionElement objects
     corresponding to the supplied dataset instances or throw exception if
     this is not a valid collection of the specified type.
     """
-    dataset_collection = model.DatasetCollection()
-    set_collection_elements(dataset_collection, type, dataset_instances)
+    dataset_collection = collection or model.DatasetCollection()
+    associated_identifiers = associated_identifiers or set()
+    set_collection_elements(dataset_collection, type, dataset_instances, associated_identifiers)
     return dataset_collection
 
 
-def set_collection_elements(dataset_collection, type, dataset_instances):
-    element_index = 0
+def set_collection_elements(dataset_collection, type, dataset_instances, associated_identifiers):
+    new_element_keys = OrderedSet(dataset_instances.keys()) - associated_identifiers
+    new_dataset_instances = {k: dataset_instances[k] for k in new_element_keys}
+    dataset_collection.element_count = dataset_collection.element_count or 0
+    element_index = dataset_collection.element_count
     elements = []
-    for element in type.generate_elements(dataset_instances):
+    for element in type.generate_elements(new_dataset_instances):
         element.element_index = element_index
         element.collection = dataset_collection
         elements.append(element)
 
         element_index += 1
+        associated_identifiers.add(element.element_identifier)
 
     dataset_collection.element_count = element_index
     return dataset_collection
@@ -33,6 +39,9 @@ class CollectionBuilder:
     def __init__(self, collection_type_description):
         self._collection_type_description = collection_type_description
         self._current_elements = {}
+        # Store collection here so we don't recreate the collection all the time
+        self.collection = None
+        self.associated_identifiers = set()
 
     def replace_elements_in_collection(self, template_collection, replacement_dict):
         self._current_elements = self._replace_elements_in_collection(
@@ -79,13 +88,15 @@ class CollectionBuilder:
             for identifier, element in elements.items():
                 new_elements[identifier] = element.build()
             elements = new_elements
+        else:
+            self._current_elements = {}
         return elements
 
     def build(self):
         type_plugin = self._collection_type_description.rank_type_plugin()
-        collection = build_collection(type_plugin, self.build_elements())
-        collection.collection_type = self._collection_type_description.collection_type
-        return collection
+        self.collection = build_collection(type_plugin, self.build_elements(), self.collection, self.associated_identifiers)
+        self.collection.collection_type = self._collection_type_description.collection_type
+        return self.collection
 
     @property
     def _subcollection_type_description(self):
@@ -107,8 +118,11 @@ class BoundCollectionBuilder(CollectionBuilder):
         collection_type_description = COLLECTION_TYPE_DESCRIPTION_FACTORY.for_collection_type(collection_type)
         super().__init__(collection_type_description)
 
-    def populate(self):
+    def populate_partial(self):
         elements = self.build_elements()
         type_plugin = self._collection_type_description.rank_type_plugin()
-        set_collection_elements(self.dataset_collection, type_plugin, elements)
+        set_collection_elements(self.dataset_collection, type_plugin, elements, self.associated_identifiers)
+
+    def populate(self):
+        self.populate_partial()
         self.dataset_collection.mark_as_populated()

lib/galaxy/model/dataset_collections/types/paired.py

@@ -4,8 +4,6 @@ from ..types import BaseDatasetCollectionType
 FORWARD_IDENTIFIER = "forward"
 REVERSE_IDENTIFIER = "reverse"
 
-INVALID_IDENTIFIERS_MESSAGE = f"Paired instance must define '{FORWARD_IDENTIFIER}' and '{REVERSE_IDENTIFIER}' datasets ."
-
 
 class PairedDatasetCollectionType(BaseDatasetCollectionType):
     """
@@ -17,20 +15,20 @@ class PairedDatasetCollectionType(BaseDatasetCollectionType):
         pass
 
     def generate_elements(self, elements):
-        forward_dataset = elements.get(FORWARD_IDENTIFIER, None)
-        reverse_dataset = elements.get(REVERSE_IDENTIFIER, None)
-        if not forward_dataset or not reverse_dataset:
-            self._validation_failed(INVALID_IDENTIFIERS_MESSAGE)
-        left_association = DatasetCollectionElement(
-            element=forward_dataset,
-            element_identifier=FORWARD_IDENTIFIER,
-        )
-        right_association = DatasetCollectionElement(
-            element=reverse_dataset,
-            element_identifier=REVERSE_IDENTIFIER,
-        )
-        yield left_association
-        yield right_association
+        forward_dataset = elements.get(FORWARD_IDENTIFIER)
+        reverse_dataset = elements.get(REVERSE_IDENTIFIER)
+        if forward_dataset:
+            left_association = DatasetCollectionElement(
+                element=forward_dataset,
+                element_identifier=FORWARD_IDENTIFIER,
+            )
+            yield left_association
+        if reverse_dataset:
+            right_association = DatasetCollectionElement(
+                element=reverse_dataset,
+                element_identifier=REVERSE_IDENTIFIER,
+            )
+            yield right_association
 
     def prototype_elements(self):
         left_association = DatasetCollectionElement(

lib/galaxy/model/mapping.py

@@ -271,6 +271,7 @@ model.HistoryDatasetAssociationHistory.table = Table(
 model.Dataset.table = Table(
     "dataset", metadata,
     Column("id", Integer, primary_key=True),
+    Column('job_id', Integer, ForeignKey('job.id'), index=True, nullable=True),
     Column("create_time", DateTime, default=now),
     Column("update_time", DateTime, index=True, default=now, onupdate=now),
     Column("state", TrimmedString(64), index=True),
@@ -1769,6 +1770,7 @@ simple_mapping(model.HistoryDatasetAssociation,
 )
 
 simple_mapping(model.Dataset,
+    job=relation(model.Job, primaryjoin=(model.Dataset.table.c.job_id == model.Job.table.c.id)),
     history_associations=relation(model.HistoryDatasetAssociation,
         primaryjoin=(model.Dataset.table.c.id == model.HistoryDatasetAssociation.table.c.dataset_id)),
     active_history_associations=relation(model.HistoryDatasetAssociation,

lib/galaxy/model/migrate/check.py

@@ -9,9 +9,9 @@ from sqlalchemy import (
     Table
 )
 from sqlalchemy.exc import NoSuchTableError
-from sqlalchemy_utils import create_database, database_exists
 
 from galaxy.model import mapping
+from galaxy.model.database_utils import create_database, database_exists
 
 log = logging.getLogger(__name__)
 

lib/galaxy/model/migrate/versions/0166_job_state_summary_view.py

@@ -15,12 +15,13 @@ def upgrade(migrate_engine):
     print(__doc__)
     # drop first because sqlite does not support or_replace
     downgrade(migrate_engine)
-    create_view = CreateView(HistoryDatasetCollectionJobStateSummary)
+    view = HistoryDatasetCollectionJobStateSummary
+    create_view = CreateView(view.name, view.__view__)
     # print(str(create_view.compile(migrate_engine)))
     migrate_engine.execute(create_view)
 
 
 def downgrade(migrate_engine):
-    drop_view = DropView(HistoryDatasetCollectionJobStateSummary)
+    drop_view = DropView(HistoryDatasetCollectionJobStateSummary.name)
     # print(str(drop_view.compile(migrate_engine)))
     migrate_engine.execute(drop_view)

lib/galaxy/model/migrate/versions/0173_add_job_id_to_dataset.py

@@ -0,0 +1,41 @@
+"""
+Migration script for adding job_id column to dataset table.
+"""
+from __future__ import print_function
+
+import logging
+
+from sqlalchemy import (
+    Column,
+    ForeignKey,
+    Integer,
+    MetaData,
+)
+
+from galaxy.model.migrate.versions.util import (
+    add_column,
+    add_index,
+    drop_column,
+    drop_index,
+)
+
+log = logging.getLogger(__name__)
+metadata = MetaData()
+
+
+def upgrade(migrate_engine):
+    print(__doc__)
+    metadata.bind = migrate_engine
+    metadata.reflect()
+
+    job_id_column = Column('job_id', Integer, ForeignKey('job.id'))
+    add_column(job_id_column, 'dataset', metadata)
+    add_index('ix_dataset_job_id', 'dataset', 'job_id', metadata)
+
+
+def downgrade(migrate_engine):
+    metadata.bind = migrate_engine
+    metadata.reflect()
+
+    drop_column('job_id', 'dataset', metadata)
+    drop_index('ix_dataset_job_id', 'dataset', 'job_id', metadata)

lib/galaxy/model/orm/engine_factory.py

@@ -3,6 +3,7 @@ import logging
 import os
 import threading
 import time
+from multiprocessing.util import register_after_fork
 
 from sqlalchemy import (
     create_engine,
@@ -87,4 +88,5 @@ def build_engine(url, engine_options, database_query_profiling_proxy=False, trac
 
     # Create the database engine
     engine = create_engine(url, **engine_options)
+    register_after_fork(engine, lambda e: e.dispose())
     return engine

lib/galaxy/model/orm/scripts.py

@@ -136,8 +136,9 @@ def get_config(argv, use_argparse=True, cwd=None):
         db_url = properties["%sdatabase_connection" % config_prefix]
     else:
         db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % os.path.join(get_data_dir(properties), default_sqlite_file)
+    install_database_connection = properties.get('install_database_connection')
 
-    return dict(db_url=db_url, repo=repo, config_file=config_file, database=database)
+    return dict(db_url=db_url, repo=repo, config_file=config_file, database=database, install_database_connection=install_database_connection)
 
 
 def manage_db():

lib/galaxy/model/store/discover.py

@@ -20,6 +20,7 @@ from galaxy.exceptions import (
 )
 from galaxy.model.dataset_collections import builder
 from galaxy.util import (
+    chunk_iterable,
     ExecutionTimer
 )
 from galaxy.util.hash_util import HASH_NAME_MAP
@@ -27,6 +28,7 @@ from galaxy.util.hash_util import HASH_NAME_MAP
 log = logging.getLogger(__name__)
 
 UNSET = object()
+DEFAULT_CHUNK_SIZE = 1000
 
 
 class ModelPersistenceContext(metaclass=abc.ABCMeta):
@@ -35,7 +37,6 @@ class ModelPersistenceContext(metaclass=abc.ABCMeta):
     This class implement the create_dataset method that takes care of populating metadata
     required for datasets and other potential model objects.
     """
-
     def create_dataset(
         self,
         ext,
@@ -57,6 +58,7 @@ class ModelPersistenceContext(metaclass=abc.ABCMeta):
         hashes=None,
         created_from_basename=None,
         final_job_state='ok',
+        creating_job_id=None,
     ):
         tag_list = tag_list or []
         sources = sources or []
@@ -88,9 +90,10 @@ class ModelPersistenceContext(metaclass=abc.ABCMeta):
                                                                       dbkey=dbkey,
                                                                       create_dataset=True,
                                                                       flush=False,
-                                                                      sa_session=sa_session)
-
+                                                                      sa_session=sa_session,
+                                                                      creating_job_id=creating_job_id)
                 self.persist_object(primary_data)
+
                 if init_from:
                     self.permission_provider.copy_dataset_permissions(init_from, primary_data)
                     primary_data.state = init_from.state
@@ -217,9 +220,20 @@ class ModelPersistenceContext(metaclass=abc.ABCMeta):
         #    <sort regex="part_(\d+)_sample_([^_]+).fastq" by="2:lexical,1:numerical" />
         if name is None:
             name = "unnamed output"
+        if self.flush_per_n_datasets and self.flush_per_n_datasets > 0:
+            for chunk in chunk_iterable(filenames.items(), size=self.flush_per_n_datasets):
+                self._populate_elements(chunk=chunk, name=name, root_collection_builder=root_collection_builder, metadata_source_name=metadata_source_name, final_job_state=final_job_state)
+                if len(chunk) == self.flush_per_n_datasets:
+                    # In most cases we don't need to flush, that happens in the caller.
+                    # Only flush here for saving memory.
+                    root_collection_builder.populate_partial()
+                    self.flush()
+        else:
+            self._populate_elements(chunk=filenames.items(), name=name, root_collection_builder=root_collection_builder, metadata_source_name=metadata_source_name, final_job_state=final_job_state)
 
+    def _populate_elements(self, chunk, name, root_collection_builder, metadata_source_name, final_job_state):
         element_datasets = {'element_identifiers': [], 'datasets': [], 'tag_lists': [], 'paths': [], 'extra_files': []}
-        for filename, discovered_file in filenames.items():
+        for filename, discovered_file in chunk:
             create_dataset_timer = ExecutionTimer()
             fields_match = discovered_file.match
             if not fields_match:
@@ -404,6 +418,7 @@ class SessionlessModelPersistenceContext(ModelPersistenceContext):
         self.sa_session = None
         self.object_store = object_store
         self.export_store = export_store
+        self.flush_per_n_datasets = None
 
         self.job_working_directory = working_directory  # TODO: rename...