opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #11720 from dannon/circus

Browse Source 
Circus + Celery - MVP
This commit is contained in:
Dannon
2021-03-30 06:08:33 -04:00
committed by GitHub
parent 1f1c6599e8 537495a4e3
commit bb395a2778
42 changed files with 459 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/dev.ini
+34
View File
@@ -0,0 +1,34 @@
[circus]
debug = True
[env]
GALAXY_CONFIG_FILE=config/galaxy.yml
PYTHONPATH=lib
[watcher:web]
cmd = gunicorn 'galaxy.webapps.galaxy.fast_factory:factory()' --pythonpath lib -k uvicorn.workers.UvicornWorker -b fd://$(circus.sockets.web)
send_hup = true
numprocesses = 1
use_sockets = True
stop_signal = TERM
stop_children = True
copy_env = True
[watcher:client]
working_dir = client
cmd = yarn watch
numprocesses = 1
singleton = True
copy_env = True
stop_signal = TERM
stop_children = True
[socket:web]
host = 0.0.0.0
port = 8080
[watcher:celery]
cmd = celery
args = --app galaxy.celery worker -l debug
copy_env = True
numprocesses = 1
doc/source/admin/galaxy_options.rst
+12
View File
@@ -4360,6 +4360,18 @@
:Type: str
~~~~~~~~~~~~~~~~~~~~~~~
``enable_celery_tasks``
~~~~~~~~~~~~~~~~~~~~~~~
:Description:
Offload long-running tasks to a Celery task queue. Activate this
only if you have setup a Celery worker for Galaxy. For details,
see https://docs.galaxyproject.org/en/master/admin/production.html
:Default: ``false``
:Type: bool
~~~~~~~~~~~~~~
``use_pbkdf2``
~~~~~~~~~~~~~~
lib/galaxy/app.py
+126 -109
View File
@@ -2,7 +2,7 @@ import logging
import signal
import sys
import time
from typing import Any
from typing import Any, Callable, List, Tuple
from sqlalchemy.orm.scoping import (
scoped_session,
@@ -70,19 +70,41 @@ from galaxy.web_stack import application_stack_instance, ApplicationStack
from galaxy.webhooks import WebhooksRegistry
from galaxy.workflow.trs_proxy import TrsProxy
from .di import Container
from .structured_app import BasicApp, StructuredApp
from .structured_app import BasicApp, MinimalManagerApp, StructuredApp
log = logging.getLogger(__name__)
app = None
class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container):
"""Encapsulates the state of a Universe application"""
class HaltableContainer(Container):
haltables: List[Tuple[str, Callable]]
def __init__(self, **kwargs) -> None:
def __init__(self) -> None:
super().__init__()
self.haltables = []
def shutdown(self):
exception = None
for what, haltable in self.haltables:
try:
haltable()
except Exception as e:
log.exception(f"Failed to shutdown {what} cleanly")
exception = exception or e
if exception is not None:
raise exception
class MinimalGalaxyApplication(BasicApp, config.ConfiguresGalaxyMixin, HaltableContainer):
"""Encapsulates the state of a minimal Galaxy application"""
def __init__(self, fsmon=False, configure_logging=True, **kwargs) -> None:
super().__init__()
self.haltables = [
("object store", self._shutdown_object_store),
("database connection", self._shutdown_model),
]
self._register_singleton(BasicApp, self)
self._register_singleton(StructuredApp, self)
if not log.handlers:
# Paste didn't handle it, so we need a temporary basic log
# configured. The handler added here gets dumped and replaced with
@@ -91,38 +113,47 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container
log.debug("python path is: %s", ", ".join(sys.path))
self.name = 'galaxy'
self.is_webapp = False
startup_timer = ExecutionTimer()
self.new_installation = False
# Read config file and check for errors
self.config: Any = self._register_singleton(config.Configuration, config.Configuration(**kwargs))
self.config.check()
config.configure_logging(self.config)
self.execution_timer_factory = self._register_singleton(ExecutionTimerFactory, ExecutionTimerFactory(self.config))
self.configure_fluent_log()
# A lot of postfork initialization depends on the server name, ensure it is set immediately after forking before other postfork functions
self.application_stack = self._register_singleton(ApplicationStack, application_stack_instance(app=self))
self.application_stack.register_postfork_function(self.application_stack.set_postfork_server_name, self)
self.config.reload_sanitize_allowlist(explicit='sanitize_allowlist_file' in kwargs)
self.amqp_internal_connection_obj = galaxy.queues.connection_from_config(self.config)
# queue_worker *can* be initialized with a queue, but here we don't
# want to and we'll allow postfork to bind and start it.
self.queue_worker = self._register_singleton(GalaxyQueueWorker, GalaxyQueueWorker(self))
self._configure_tool_shed_registry()
if configure_logging:
config.configure_logging(self.config)
self._configure_object_store(fsmon=True)
# Setup the database engine and ORM
config_file = kwargs.get('global_conf', {}).get('__file__', None)
if config_file:
log.debug('Using "galaxy.ini" config file: %s', config_file)
check_migrate_tools = self.config.check_migrate_tools
self._configure_models(check_migrate_databases=self.config.check_migrate_databases, check_migrate_tools=check_migrate_tools, config_file=config_file)
# Security helper
self._configure_security()
self._register_singleton(IdEncodingHelper, self.security)
self._register_singleton(SharedModelMapping, self.model)
self._register_singleton(GalaxyModelMapping, self.model)
self._register_singleton(scoped_session, self.model.context)
def configure_fluent_log(self):
if self.config.fluent_log:
from galaxy.util.custom_logging.fluent_log import FluentTraceLogger
self.trace_logger = FluentTraceLogger('galaxy', self.config.fluent_host, self.config.fluent_port)
else:
self.trace_logger = None
def _shutdown_object_store(self):
self.object_store.shutdown()
def _shutdown_model(self):
self.model.engine.dispose()
class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication):
"""Extends the MinimalGalaxyApplication with most managers that are not tied to a web or job handling context."""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._register_singleton(MinimalManagerApp, self)
self.execution_timer_factory = self._register_singleton(ExecutionTimerFactory, ExecutionTimerFactory(self.config))
self.configure_fluent_log()
# Tag handler
self.tag_handler = self._register_singleton(GalaxyTagHandler)
self.user_manager = self._register_singleton(UserManager)
@@ -133,17 +164,66 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container
self.dataset_collections_service = self._register_singleton(DatasetCollectionManager)
self.workflow_manager = self._register_singleton(WorkflowsManager)
self.workflow_contents_manager = self._register_singleton(WorkflowContentsManager)
self.dependency_resolvers_view = self._register_singleton(DependencyResolversView, DependencyResolversView(self))
self.test_data_resolver = self._register_singleton(TestDataResolver, TestDataResolver(file_dirs=self.config.tool_test_data_directories))
self.library_folder_manager = self._register_singleton(FolderManager)
self.library_manager = self._register_singleton(LibraryManager)
self.role_manager = self._register_singleton(RoleManager)
self.dynamic_tool_manager = self._register_singleton(DynamicToolManager)
self.api_keys_manager = self._register_singleton(ApiKeyManager)
# ConfiguredFileSources
self.file_sources = self._register_singleton(ConfiguredFileSources, ConfiguredFileSources.from_app_config(self.config))
# We need the datatype registry for running certain tasks that modify HDAs, and to build the registry we need
# to setup the installed repositories ... this is not ideal
self._configure_tool_config_files()
self.installed_repository_manager = self._register_singleton(InstalledRepositoryManager, InstalledRepositoryManager(self))
self._configure_datatypes_registry(self.installed_repository_manager)
self._register_singleton(Registry, self.datatypes_registry)
galaxy.model.set_datatypes_registry(self.datatypes_registry)
self.sentry_client = None
if self.config.sentry_dsn:
def postfork_sentry_client():
import raven
self.sentry_client = raven.Client(self.config.sentry_dsn, transport=raven.transport.HTTPTransport)
self.application_stack.register_postfork_function(postfork_sentry_client)
class UniverseApplication(StructuredApp, GalaxyManagerApplication):
"""Encapsulates the state of a Universe application"""
def __init__(self, **kwargs) -> None:
startup_timer = ExecutionTimer()
super().__init__(fsmon=True, **kwargs)
self.haltables = [
("queue worker", self._shutdown_queue_worker),
("file watcher", self._shutdown_watcher),
("database heartbeat", self._shutdown_database_heartbeat),
("workflow scheduler", self._shutdown_scheduling_manager),
("object store", self._shutdown_object_store),
("job manager", self._shutdown_job_manager),
("application heartbeat", self._shutdown_heartbeat),
("repository manager", self._shutdown_repo_manager),
("database connection", self._shutdown_model),
("application stack", self._shutdown_application_stack),
]
self._register_singleton(StructuredApp, self)
# A lot of postfork initialization depends on the server name, ensure it is set immediately after forking before other postfork functions
self.application_stack = self._register_singleton(ApplicationStack, application_stack_instance(app=self))
self.application_stack.register_postfork_function(self.application_stack.set_postfork_server_name, self)
self.config.reload_sanitize_allowlist(explicit='sanitize_allowlist_file' in kwargs)
self.amqp_internal_connection_obj = galaxy.queues.connection_from_config(self.config)
# queue_worker *can* be initialized with a queue, but here we don't
# want to and we'll allow postfork to bind and start it.
self.queue_worker = self._register_singleton(GalaxyQueueWorker, GalaxyQueueWorker(self))
self._configure_tool_shed_registry()
self.dependency_resolvers_view = self._register_singleton(DependencyResolversView, DependencyResolversView(self))
self.test_data_resolver = self._register_singleton(TestDataResolver, TestDataResolver(file_dirs=self.config.tool_test_data_directories))
self.dynamic_tool_manager = self._register_singleton(DynamicToolManager)
self.api_keys_manager = self._register_singleton(ApiKeyManager)
# Tool Data Tables
self._configure_tool_data_tables(from_shed_config=False)
# Load dbkey / genome build manager
@@ -169,14 +249,7 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container
self.tool_shed_repository_cache = self._register_singleton(ToolShedRepositoryCache)
# Watch various config files for immediate reload
self.watchers = self._register_singleton(ConfigWatchers)
self._configure_tool_config_files()
self.installed_repository_manager = self._register_singleton(InstalledRepositoryManager, InstalledRepositoryManager(self))
self._configure_datatypes_registry(self.installed_repository_manager)
self._register_singleton(Registry, self.datatypes_registry)
galaxy.model.set_datatypes_registry(self.datatypes_registry)
self._configure_toolbox()
# Load Data Manager
self.data_managers = self._register_singleton(DataManagers)
# Load the update repository manager.
@@ -231,16 +304,6 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container
self.authnz_manager = managers.AuthnzManager(self,
self.config.oidc_config_file,
self.config.oidc_backends_config_file)
self.sentry_client = None
if self.config.sentry_dsn:
def postfork_sentry_client():
import raven
self.sentry_client = raven.Client(self.config.sentry_dsn, transport=raven.transport.HTTPTransport)
self.application_stack.register_postfork_function(postfork_sentry_client)
# Start the job manager
from galaxy.jobs import manager
self.job_manager = self._register_singleton(manager.JobManager)
@@ -284,8 +347,6 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container
# Delay toolbox index until after startup
self.application_stack.register_postfork_function(lambda: send_local_control_task(self, 'rebuild_toolbox_search_index'))
self.model.engine.dispose()
# Inject url_for for components to more easily optionally depend
# on url_for.
self.url_for = url_for
@@ -293,74 +354,30 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container
self.server_starttime = int(time.time()) # used for cachebusting
log.info("Galaxy app startup finished %s" % startup_timer)
def shutdown(self):
log.debug('Shutting down')
exception = None
try:
self.queue_worker.shutdown()
except Exception as e:
exception = exception or e
log.exception("Failed to shutdown control worker cleanly")
try:
self.watchers.shutdown()
except Exception as e:
exception = exception or e
log.exception("Failed to shutdown configuration watchers cleanly")
try:
self.database_heartbeat.shutdown()
except Exception as e:
exception = exception or e
log.exception("Failed to shutdown database heartbeat cleanly")
try:
self.workflow_scheduling_manager.shutdown()
except Exception as e:
exception = exception or e
log.exception("Failed to shutdown workflow scheduling manager cleanly")
try:
self.job_manager.shutdown()
except Exception as e:
exception = exception or e
log.exception("Failed to shutdown job manager cleanly")
try:
self.object_store.shutdown()
except Exception as e:
exception = exception or e
log.exception("Failed to shutdown object store cleanly")
try:
if self.heartbeat:
self.heartbeat.shutdown()
except Exception as e:
exception = exception or e
log.exception("Failed to shutdown heartbeat cleanly")
try:
self.update_repository_manager.shutdown()
except Exception as e:
exception = exception or e
log.exception("Failed to shutdown update repository manager cleanly")
def _shutdown_queue_worker(self):
self.queue_worker.shutdown()
try:
self.model.engine.dispose()
except Exception as e:
exception = exception or e
log.exception("Failed to shutdown SA database engine cleanly")
def _shutdown_watcher(self):
self.watchers.shutdown()
try:
self.application_stack.shutdown()
except Exception as e:
exception = exception or e
log.exception("Failed to shutdown application stack interface cleanly")
def _shutdown_database_heartbeat(self):
self.database_heartbeat.shutdown()
if exception:
raise exception
else:
log.debug('Finished shutting down')
def _shutdown_scheduling_manager(self):
self.workflow_scheduling_manager.shutdown()
def configure_fluent_log(self):
if self.config.fluent_log:
from galaxy.util.custom_logging.fluent_log import FluentTraceLogger
self.trace_logger = FluentTraceLogger('galaxy', self.config.fluent_host, self.config.fluent_port)
else:
self.trace_logger = None
def _shutdown_job_manager(self):
self.job_manager.shutdown()
def _shutdown_heartbeat(self):
if self.heartbeat:
self.heartbeat.shutdown()
def _shutdown_repo_manager(self):
self.update_repository_manager.shutdown()
def _shutdown_application_stack(self):
self.application_stack.shutdown()
@property
def is_job_handler(self) -> bool:
lib/galaxy/celery/__init__.py
+55
View File
@@ -0,0 +1,55 @@
import os
from functools import lru_cache
from celery import Celery
from galaxy.config import Configuration
from galaxy.util.custom_logging import get_logger
from galaxy.util.properties import load_app_properties
log = get_logger(__name__)
@lru_cache(maxsize=1)
def get_galaxy_app():
import galaxy.app
if galaxy.app.app:
return galaxy.app.app
kwargs = get_app_properties()
if kwargs:
kwargs['check_migrate_tools'] = False
kwargs['check_migrate_databases'] = False
galaxy_app = galaxy.app.GalaxyManagerApplication(configure_logging=False, **kwargs)
return galaxy_app
@lru_cache(maxsize=1)
def get_app_properties():
config_file = os.environ.get("GALAXY_CONFIG_FILE")
if config_file:
return load_app_properties(
config_file=os.path.abspath(config_file),
config_section='galaxy',
)
@lru_cache(maxsize=1)
def get_config():
kwargs = get_app_properties()
if kwargs:
kwargs['override_tempdir'] = False
return Configuration(**kwargs)
def get_broker():
config = get_config()
if config:
return config.amqp_internal_connection
broker = get_broker()
celery_app = Celery('galaxy', broker=broker, include=['galaxy.celery.tasks'])
if __name__ == '__main__':
celery_app.start()
lib/galaxy/celery/tasks.py
+51
View File
@@ -0,0 +1,51 @@
from lagom import magic_bind_to_container
from sqlalchemy.orm.scoping import (
scoped_session,
)
from galaxy.celery import celery_app
from galaxy.managers.hdas import HDAManager
from galaxy.managers.lddas import LDDAManager
from galaxy.model import User
from galaxy.util.custom_logging import get_logger
from . import get_galaxy_app
log = get_logger(__name__)
def galaxy_task(func):
app = get_galaxy_app()
if app:
return magic_bind_to_container(app)(func)
return func
@celery_app.task(ignore_result=True)
@galaxy_task
def recalculate_user_disk_usage(session: scoped_session, user_id=None):
if user_id:
user = session.query(User).get(user_id)
if user:
user.calculate_and_set_disk_usage()
log.info(f"New user disk usage is {user.disk_usage}")
else:
log.error("Recalculate user disk usage task failed, user %s not found" % user_id)
else:
log.error("Recalculate user disk usage task received without user_id.")
@celery_app.task(ignore_result=True)
@galaxy_task
def purge_hda(hda_manager: HDAManager, hda_id):
hda = hda_manager.by_id(hda_id)
hda_manager._purge(hda)
@celery_app.task
@galaxy_task
def set_metadata(hda_manager: HDAManager, ldda_manager: LDDAManager, dataset_id, model_class='HistoryDatasetAssociation'):
if model_class == 'HistoryDatasetAssociation':
dataset = hda_manager.by_id(dataset_id)
elif model_class == 'LibraryDatasetDatasetAssociation':
dataset = ldda_manager.by_id(dataset_id)
dataset.datatype.set_meta(dataset)
lib/galaxy/config/sample/galaxy.yml.sample
+5
View File
@@ -2159,6 +2159,11 @@ galaxy:
# commented out line below).
#amqp_internal_connection: sqlalchemy+sqlite:///./database/control.sqlite?isolation_level=IMMEDIATE
# Offload long-running tasks to a Celery task queue. Activate this
# only if you have setup a Celery worker for Galaxy. For details, see
# https://docs.galaxyproject.org/en/master/admin/production.html
#enable_celery_tasks: false
# Allow disabling pbkdf2 hashing of passwords for legacy situations.
# This should normally be left enabled unless there is a specific
# reason to disable it.
lib/galaxy/dependencies/dev-requirements.txt
+13 -4
View File
@@ -17,6 +17,7 @@ bagit==1.8.1; python_version >= "3.6" and python_full_version < "3.0.0" and pyth
bcrypt==3.2.0; python_version >= "3.6"
bdbag==1.6.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0" and python_version < "4")
beaker==1.11.0
billiard==3.6.3.0; python_version >= "3.6"
bioblend==0.15.0; python_version >= "3.6"
bleach==3.3.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
boltons==20.2.1
@@ -27,11 +28,16 @@ bx-python==0.8.11; python_version >= "3.6"
cachecontrol==0.11.7; python_version >= "3.6" and python_version < "4"
cached-property==1.5.2; python_version < "3.8" and python_version >= "3.6"
cachetools==4.2.1; python_version >= "3.5" and python_version < "4.0" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0")
celery==5.0.5; python_version >= "3.6"
certifi==2020.12.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6"
cffi==1.14.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
cffi==1.14.5; implementation_name == "pypy" and python_version >= "3.6" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0")
chardet==4.0.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4"
cheetah3==3.2.6.post1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
click==7.1.2; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
circus==0.17.1
click-didyoumean==0.0.3; python_version >= "3.6"
click-plugins==1.1.1; python_version >= "3.6"
click-repl==0.1.6; python_version >= "3.6"
click==7.1.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
cliff==3.7.0; python_version >= "3.6"
cloudauthz==0.6.0
cloudbridge==2.1.0
@@ -132,14 +138,15 @@ pbr==5.5.1; python_version >= "3.6"
pluggy==0.13.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
port-for==0.4; python_version >= "3.6"
prettytable==0.7.2; python_version >= "3.6"
prompt-toolkit==3.0.3; python_version >= "3.6"
protobuf==3.15.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
prov==1.5.1; python_version >= "3.6" and python_version < "4"
psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
pulsar-galaxy-lib==0.14.2
py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or python_full_version >= "3.4.0" and python_version >= "3.6" and implementation_name == "pypy"
pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
pyasn1==0.4.8; python_version >= "3.5" and python_version < "4"
pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.6" and python_full_version >= "3.4.0"
pycryptodome==3.10.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
pydantic==1.7.3; python_version >= "3.6" and python_version < "4.0"
pydot==1.4.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.4.0"
@@ -178,6 +185,7 @@ python3-openid==3.2.0; python_version >= "3.0"
pytz==2021.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6"
pyuwsgi==2.0.19.1.post0
pyyaml==5.4.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
pyzmq==22.0.3; python_version >= "3.6"
rdflib-jsonld==0.5.0; python_version >= "3.6" and python_version < "4"
rdflib==4.2.2; python_version >= "3.6" and python_version < "4"
recommonmark==0.7.1
@@ -226,6 +234,7 @@ tempita==0.5.2
tenacity==7.0.0
testfixtures==6.17.1
toml==0.10.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6"
tornado==6.1; python_version >= "3.5"
tqdm==4.59.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
twill==3.0
typing-extensions==3.7.4.3; python_version >= "3.6" and python_version < "3.8"
lib/galaxy/dependencies/pinned-requirements.txt
+13 -3
View File
@@ -15,6 +15,7 @@ bagit==1.8.1; python_version >= "3.6" and python_full_version < "3.0.0" and pyth
bcrypt==3.2.0; python_version >= "3.6"
bdbag==1.6.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0" and python_version < "4")
beaker==1.11.0
billiard==3.6.3.0; python_version >= "3.6"
bioblend==0.15.0; python_version >= "3.6"
bleach==3.3.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
boltons==20.2.1
@@ -25,11 +26,16 @@ bx-python==0.8.11; python_version >= "3.6"
cachecontrol==0.11.7; python_version >= "3.6" and python_version < "4"
cached-property==1.5.2; python_version < "3.8" and python_version >= "3.6"
cachetools==4.2.1; python_version >= "3.5" and python_version < "4.0" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0")
celery==5.0.5; python_version >= "3.6"
certifi==2020.12.5; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4"
cffi==1.14.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
cffi==1.14.5; implementation_name == "pypy" and python_version >= "3.6" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0")
chardet==4.0.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4"
cheetah3==3.2.6.post1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
click==7.1.2; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
circus==0.17.1
click-didyoumean==0.0.3; python_version >= "3.6"
click-plugins==1.1.1; python_version >= "3.6"
click-repl==0.1.6; python_version >= "3.6"
click==7.1.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
cliff==3.7.0; python_version >= "3.6"
cloudauthz==0.6.0
cloudbridge==2.1.0
@@ -115,13 +121,15 @@ paste==3.5.0
pastedeploy==2.1.1
pbr==5.5.1; python_version >= "3.6"
prettytable==0.7.2; python_version >= "3.6"
prompt-toolkit==3.0.3; python_version >= "3.6"
protobuf==3.15.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
prov==1.5.1; python_version >= "3.6" and python_version < "4"
psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
pulsar-galaxy-lib==0.14.2
py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.6" and python_full_version >= "3.4.0"
pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
pyasn1==0.4.8; python_version >= "3.5" and python_version < "4"
pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.6" and python_full_version >= "3.4.0"
pycryptodome==3.10.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
pydantic==1.7.3; python_version >= "3.6" and python_version < "4.0"
pydot==1.4.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.4.0"
@@ -147,6 +155,7 @@ python3-openid==3.2.0; python_version >= "3.0"
pytz==2021.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6"
pyuwsgi==2.0.19.1.post0
pyyaml==5.4.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
pyzmq==22.0.3; python_version >= "3.6"
rdflib-jsonld==0.5.0; python_version >= "3.6" and python_version < "4"
rdflib==4.2.2; python_version >= "3.6" and python_version < "4"
refgenconf==0.9.3
@@ -178,6 +187,7 @@ stevedore==3.3.0; python_version >= "3.6"
svgwrite==1.4.1; python_version >= "3.6"
tempita==0.5.2
tenacity==7.0.0
tornado==6.1; python_version >= "3.5"
tqdm==4.59.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
typing-extensions==3.7.4.3; python_version >= "3.6" and python_version < "3.8"
tzlocal==2.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4"
lib/galaxy/jobs/__init__.py
+2 -2
View File
@@ -57,7 +57,7 @@ from galaxy.jobs.runners import BaseJobRunner, JobState
from galaxy.metadata import get_metadata_compute_strategy
from galaxy.model import store
from galaxy.objectstore import ObjectStorePopulator
from galaxy.structured_app import StructuredApp
from galaxy.structured_app import MinimalManagerApp
from galaxy.tool_util.deps import requirements
from galaxy.tool_util.output_checker import (
check_output,
@@ -301,7 +301,7 @@ class JobConfiguration(ConfiguresHandlers):
<when value="yes"/>
</conditional>"""
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
"""Parse the job configuration XML.
"""
self.app = app
lib/galaxy/jobs/manager.py
+2 -2
View File
@@ -9,7 +9,7 @@ from sqlalchemy.sql.expression import null
from galaxy.exceptions import HandlerAssignmentError, ToolExecutionError
from galaxy.jobs import handler, NoopQueue
from galaxy.model import Job
from galaxy.structured_app import StructuredApp
from galaxy.structured_app import MinimalManagerApp
from galaxy.web_stack.message import JobHandlerMessage
log = logging.getLogger(__name__)
@@ -21,7 +21,7 @@ class JobManager:
"""
job_handler: handler.JobHandlerI
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
self.app = app
self.job_lock = False
if self.app.is_job_handler:
lib/galaxy/managers/base.py
+5 -5
View File
@@ -37,7 +37,7 @@ from sqlalchemy.orm.scoping import scoped_session
from galaxy import exceptions
from galaxy import model
from galaxy.model import tool_shed_install
from galaxy.structured_app import BasicApp, StructuredApp
from galaxy.structured_app import BasicApp, MinimalManagerApp
from galaxy.util import namedtuple
log = logging.getLogger(__name__)
@@ -488,7 +488,7 @@ class HasAModelManager:
# examples where this doesn't really work are ConfigurationSerializer (no manager)
# and contents (2 managers)
def __init__(self, app: StructuredApp, manager=None, **kwargs):
def __init__(self, app: MinimalManagerApp, manager=None, **kwargs):
self._manager = manager
@property
@@ -542,7 +542,7 @@ class ModelSerializer(HasAModelManager):
default_view: Optional[str]
views: Dict[str, List[str]]
def __init__(self, app: StructuredApp, **kwargs):
def __init__(self, app: MinimalManagerApp, **kwargs):
"""
Set up serializer map, any additional serializable keys, and views here.
"""
@@ -713,7 +713,7 @@ class ModelDeserializer(HasAModelManager):
"""
# TODO:?? a larger question is: which should be first? Deserialize then validate - or - validate then deserialize?
def __init__(self, app: StructuredApp, validator=None, **kwargs):
def __init__(self, app: MinimalManagerApp, validator=None, **kwargs):
"""
Set up deserializers and validator.
"""
@@ -904,7 +904,7 @@ class ModelFilterParser(HasAModelManager):
orm_filter_parsers: Dict[str, Dict]
fn_filter_parsers: Dict[str, Dict]
def __init__(self, app: StructuredApp, **kwargs):
def __init__(self, app: MinimalManagerApp, **kwargs):
"""
Set up serializer map, any additional serializable keys, and views here.
"""
lib/galaxy/managers/configuration.py
+2 -2
View File
@@ -15,7 +15,7 @@ from typing import (
List,
)
from galaxy.app import StructuredApp
from galaxy.app import MinimalManagerApp
from galaxy.managers import base
from galaxy.managers.context import ProvidesUserContext
from galaxy.schema.fields import EncodedDatabaseIdField
@@ -30,7 +30,7 @@ VERSION_JSON_FILE = 'version.json'
class ConfigurationManager:
"""Interface/service object for interacting with configuration and related data."""
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
self._app = app
def get_configuration(
lib/galaxy/managers/context.py
+2 -2
View File
@@ -50,7 +50,7 @@ from galaxy.model import (
)
from galaxy.model.base import ModelMapping
from galaxy.security.idencoding import IdEncodingHelper
from galaxy.structured_app import StructuredApp
from galaxy.structured_app import MinimalManagerApp
from galaxy.util import bunch
@@ -62,7 +62,7 @@ class ProvidesAppContext:
"""
@abc.abstractproperty
def app(self) -> StructuredApp:
def app(self) -> MinimalManagerApp:
"""Provide access to the Galaxy ``app`` object.
"""
lib/galaxy/managers/datasets.py
+7 -5
View File
@@ -18,7 +18,7 @@ from galaxy.managers import (
secured,
users
)
from galaxy.structured_app import StructuredApp
from galaxy.structured_app import MinimalManagerApp
from galaxy.util.checkers import check_binary
log = logging.getLogger(__name__)
@@ -33,7 +33,7 @@ class DatasetManager(base.ModelManager, secured.AccessibleManagerMixin, deletabl
# TODO:?? get + error_if_uploading is common pattern, should upload check be worked into access/owed?
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
super().__init__(app)
self.permissions = DatasetRBACPermissions(app)
# needed for admin test
@@ -143,7 +143,7 @@ class DatasetRBACPermissions:
class DatasetSerializer(base.ModelSerializer, deletable.PurgableSerializerMixin):
model_manager_class = DatasetManager
def __init__(self, app: StructuredApp, user_manager: users.UserManager):
def __init__(self, app: MinimalManagerApp, user_manager: users.UserManager):
super().__init__(app)
self.dataset_manager = self.manager
# needed for admin test
@@ -302,8 +302,10 @@ class DatasetAssociationManager(base.ModelManager,
if not job.finished:
# Are *all* of the job's other output datasets deleted?
if job.check_if_output_datasets_deleted():
job.mark_deleted(self.app.config.track_jobs_in_database)
self.app.job_manager.stop(job)
track_jobs_in_database = self.app.config.track_jobs_in_database
job.mark_deleted(track_jobs_in_database)
if not track_jobs_in_database:
self.app.job_manager.stop(job)
return True
return False
lib/galaxy/managers/group_roles.py
+2 -2
View File
@@ -7,7 +7,7 @@ from typing import (
)
from galaxy import model
from galaxy.app import StructuredApp
from galaxy.app import MinimalManagerApp
from galaxy.exceptions import (
ObjectNotFound,
)
@@ -22,7 +22,7 @@ log = logging.getLogger(__name__)
class GroupRolesManager:
"""Interface/service object shared by controllers for interacting with group roles."""
def __init__(self, app: StructuredApp) -> None:
def __init__(self, app: MinimalManagerApp) -> None:
self._app = app
def index(self, trans: ProvidesAppContext, group_id: EncodedDatabaseIdField) -> List[Dict[str, Any]]:
lib/galaxy/managers/group_users.py
+2 -2
View File
@@ -7,7 +7,7 @@ from typing import (
)
from galaxy import model
from galaxy.app import StructuredApp
from galaxy.app import MinimalManagerApp
from galaxy.exceptions import (
ObjectNotFound,
)
@@ -22,7 +22,7 @@ log = logging.getLogger(__name__)
class GroupUsersManager:
"""Interface/service object shared by controllers for interacting with group users."""
def __init__(self, app: StructuredApp) -> None:
def __init__(self, app: MinimalManagerApp) -> None:
self._app = app
def index(self, trans: ProvidesAppContext, group_id: EncodedDatabaseIdField) -> List[Dict[str, Any]]:
lib/galaxy/managers/groups.py
+2 -2
View File
@@ -7,7 +7,7 @@ from typing import (
from sqlalchemy import false
from galaxy import model
from galaxy.app import StructuredApp
from galaxy.app import MinimalManagerApp
from galaxy.exceptions import (
Conflict,
ObjectAttributeMissingException,
@@ -22,7 +22,7 @@ from galaxy.web import url_for
class GroupsManager:
"""Interface/service object shared by controllers for interacting with groups."""
def __init__(self, app: StructuredApp) -> None:
def __init__(self, app: MinimalManagerApp) -> None:
self._app = app
def index(self, trans: ProvidesAppContext):
lib/galaxy/managers/hdas.py
+11 -5
View File
@@ -20,7 +20,7 @@ from galaxy.managers import (
taggable,
users,
)
from galaxy.structured_app import StructuredApp
from galaxy.structured_app import MinimalManagerApp
log = logging.getLogger(__name__)
@@ -41,7 +41,7 @@ class HDAManager(datasets.DatasetAssociationManager,
# TODO: move what makes sense into DatasetManager
# TODO: which of these are common with LDDAs and can be pushed down into DatasetAssociationManager?
def __init__(self, app: StructuredApp, user_manager: users.UserManager):
def __init__(self, app: MinimalManagerApp, user_manager: users.UserManager):
"""
Set up and initialize other managers needed by hdas.
"""
@@ -135,6 +135,13 @@ class HDAManager(datasets.DatasetAssociationManager,
# .... deletion and purging
def purge(self, hda, flush=True):
if self.app.config.enable_celery_tasks:
from galaxy.celery.tasks import purge_hda
purge_hda.delay(hda_id=hda.id)
else:
self._purge(hda, flush=flush)
def _purge(self, hda, flush=True):
"""
Purge this HDA and the dataset underlying it.
"""
@@ -146,7 +153,6 @@ class HDAManager(datasets.DatasetAssociationManager,
# decrease the user's space used
if quota_amount_reduction:
user.adjust_total_disk_usage(-quota_amount_reduction)
return hda
# .... states
def error_if_uploading(self, hda):
@@ -248,7 +254,7 @@ class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerialize
annotatable.AnnotatableSerializerMixin):
model_manager_class = HDAManager
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
super().__init__(app)
self.hda_manager = self.manager
@@ -507,7 +513,7 @@ class HDADeserializer(datasets.DatasetAssociationDeserializer,
"""
model_manager_class = HDAManager
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
super().__init__(app)
self.hda_manager = self.manager
lib/galaxy/managers/hdcas.py
+5 -5
View File
@@ -16,7 +16,7 @@ from galaxy.managers import (
taggable
)
from galaxy.managers.collections_util import get_hda_and_element_identifiers
from galaxy.structured_app import StructuredApp
from galaxy.structured_app import MinimalManagerApp
from galaxy.util.zipstream import ZipstreamWrapper
@@ -87,7 +87,7 @@ class DCESerializer(base.ModelSerializer):
Serializer for DatasetCollectionElements.
"""
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
super().__init__(app)
self.hda_serializer = hdas.HDASerializer(app)
self.dc_serializer = DCSerializer(app, dce_serializer=self)
@@ -121,7 +121,7 @@ class DCSerializer(base.ModelSerializer):
Serializer for DatasetCollections.
"""
def __init__(self, app: StructuredApp, dce_serializer=None):
def __init__(self, app: MinimalManagerApp, dce_serializer=None):
super().__init__(app)
self.dce_serializer = dce_serializer or DCESerializer(app)
@@ -160,7 +160,7 @@ class DCASerializer(base.ModelSerializer):
Base (abstract) Serializer class for HDCAs and LDCAs.
"""
def __init__(self, app: StructuredApp, dce_serializer=None):
def __init__(self, app: MinimalManagerApp, dce_serializer=None):
super().__init__(app)
self.dce_serializer = dce_serializer or DCESerializer(app)
@@ -215,7 +215,7 @@ class HDCASerializer(
Serializer for HistoryDatasetCollectionAssociations.
"""
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
super().__init__(app)
self.hdca_manager = HDCAManager(app)
lib/galaxy/managers/histories.py
+6 -6
View File
@@ -23,7 +23,7 @@ from galaxy.managers import (
sharable
)
from galaxy.schema.fields import EncodedDatabaseIdField
from galaxy.structured_app import StructuredApp
from galaxy.structured_app import MinimalManagerApp
log = logging.getLogger(__name__)
@@ -40,7 +40,7 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix
# TODO: incorporate imp/exp (or alias to)
def __init__(self, app: StructuredApp, hda_manager: hdas.HDAManager, contents_manager: history_contents.HistoryContentsManager, contents_filters: history_contents.HistoryContentsFilters):
def __init__(self, app: MinimalManagerApp, hda_manager: hdas.HDAManager, contents_manager: history_contents.HistoryContentsManager, contents_filters: history_contents.HistoryContentsFilters):
super().__init__(app)
self.hda_manager = hda_manager
self.contents_manager = contents_manager
@@ -174,7 +174,7 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix
class HistoryExportView:
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
self.app = app
def get_exports(self, trans, history_id):
@@ -226,7 +226,7 @@ class HistorySerializer(sharable.SharableModelSerializer, deletable.PurgableSeri
model_manager_class = HistoryManager
SINGLE_CHAR_ABBR = 'h'
def __init__(self, app: StructuredApp, hda_manager: hdas.HDAManager, hda_serializer: hdas.HDASerializer, history_contents_serializer: history_contents.HistoryContentsSerializer):
def __init__(self, app: MinimalManagerApp, hda_manager: hdas.HDAManager, hda_serializer: hdas.HDASerializer, history_contents_serializer: history_contents.HistoryContentsSerializer):
super().__init__(app)
self.history_manager = self.manager
@@ -445,7 +445,7 @@ class HistoryDeserializer(sharable.SharableModelDeserializer, deletable.Purgable
"""
model_manager_class = HistoryManager
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
super().__init__(app)
self.history_manager = self.manager
@@ -482,7 +482,7 @@ class HistoriesService:
and pydantic models to declare its parameters and return types.
"""
def __init__(self, app: StructuredApp, manager: HistoryManager, serializer: HistorySerializer):
def __init__(self, app: MinimalManagerApp, manager: HistoryManager, serializer: HistorySerializer):
self.app = app
self.manager = manager
self.serializer = serializer
lib/galaxy/managers/history_contents.py
+3 -3
View File
@@ -32,7 +32,7 @@ from galaxy.managers import (
taggable,
tools
)
from galaxy.structured_app import StructuredApp
from galaxy.structured_app import MinimalManagerApp
log = logging.getLogger(__name__)
@@ -73,7 +73,7 @@ class HistoryContentsManager(containers.ContainerManagerMixin):
)
default_order_by = 'hid'
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
self.app = app
self.contained_manager = app[self.contained_class_manager_class]
self.subcontainer_manager = app[self.subcontainer_class_manager_class]
@@ -427,7 +427,7 @@ class HistoryContentsSerializer(base.ModelSerializer, deletable.PurgableSerializ
"""
model_manager_class = HistoryContentsManager
def __init__(self, app: StructuredApp, **kwargs):
def __init__(self, app: MinimalManagerApp, **kwargs):
super().__init__(app, **kwargs)
self.default_view = 'summary'
lib/galaxy/managers/jobs.py
+7 -5
View File
@@ -9,6 +9,7 @@ from pydantic import (
)
from sqlalchemy import and_, false, func, or_
from sqlalchemy.orm import aliased
from sqlalchemy.orm.scoping import scoped_session
from sqlalchemy.sql import select
from galaxy import model
@@ -21,6 +22,7 @@ from galaxy.managers.collections import DatasetCollectionManager
from galaxy.managers.datasets import DatasetManager
from galaxy.managers.hdas import HDAManager
from galaxy.managers.lddas import LDDAManager
from galaxy.security.idencoding import IdEncodingHelper
from galaxy.structured_app import StructuredApp
from galaxy.util import (
defaultdict,
@@ -96,17 +98,17 @@ class JobSearch:
"""Search for jobs using tool inputs or other jobs"""
def __init__(
self,
app: StructuredApp,
sa_session: scoped_session,
hda_manager: HDAManager,
dataset_collection_manager: DatasetCollectionManager,
ldda_manager: LDDAManager
ldda_manager: LDDAManager,
id_encoding_helper: IdEncodingHelper,
):
self.app = app
self.sa_session = app.model.context
self.sa_session = sa_session
self.hda_manager = hda_manager
self.dataset_collection_manager = dataset_collection_manager
self.ldda_manager = ldda_manager
self.decode_id = self.app.security.decode_id
self.decode_id = id_encoding_helper.decode_id
def by_tool_input(self, trans, tool_id, tool_version, param=None, param_dump=None, job_state='ok'):
"""Search for jobs producing same results using the 'inputs' part of a tool POST."""
lib/galaxy/managers/lddas.py
+2 -2
View File
@@ -3,7 +3,7 @@ import logging
from galaxy import model, util
from galaxy.managers import base as manager_base
from galaxy.managers.datasets import DatasetAssociationManager
from galaxy.structured_app import StructuredApp
from galaxy.structured_app import MinimalManagerApp
log = logging.getLogger(__name__)
@@ -14,7 +14,7 @@ class LDDAManager(DatasetAssociationManager):
"""
model_class = model.LibraryDatasetDatasetAssociation
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
"""
Set up and initialize other managers needed by lddas.
"""
lib/galaxy/managers/metrics.py
+2 -2
View File
@@ -13,7 +13,7 @@ from pydantic import (
Field,
)
from galaxy.app import StructuredApp
from galaxy.app import MinimalManagerApp
log = logging.getLogger(__name__)
@@ -72,7 +72,7 @@ TimeSeriesTupleGenerator = Generator[TimeSeriesTuple, None, None]
class MetricsManager:
"""Interface/service object shared by controllers for interacting with metrics."""
def __init__(self, app: StructuredApp) -> None:
def __init__(self, app: MinimalManagerApp) -> None:
self._app = app
#: set to true to send additional debugging info to the log
self.debugging = True
lib/galaxy/managers/pages.py
+5 -5
View File
@@ -32,7 +32,7 @@ from galaxy.managers.markdown_util import (
)
from galaxy.model.item_attrs import UsesAnnotations
from galaxy.schema.fields import EncodedDatabaseIdField
from galaxy.structured_app import StructuredApp
from galaxy.structured_app import MinimalManagerApp
from galaxy.util import unicodify
from galaxy.util.sanitize_html import sanitize_html
@@ -197,7 +197,7 @@ class PagesService:
and pydantic models to declare its parameters and return types.
"""
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
self.manager = PageManager(app)
self.serializer = PageSerializer(app)
self.shareable_service = sharable.ShareableService(self.manager, self.serializer)
@@ -304,7 +304,7 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations):
annotation_assoc = model.PageAnnotationAssociation
rating_assoc = model.PageRatingAssociation
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
"""
"""
super().__init__(app)
@@ -427,7 +427,7 @@ class PageSerializer(sharable.SharableModelSerializer):
model_manager_class = PageManager
SINGLE_CHAR_ABBR = 'p'
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
super().__init__(app)
self.page_manager = PageManager(app)
@@ -448,7 +448,7 @@ class PageDeserializer(sharable.SharableModelDeserializer):
"""
model_manager_class = PageManager
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
super().__init__(app)
self.page_manager = self.manager
lib/galaxy/managers/remote_files.py
+2 -2
View File
@@ -12,7 +12,7 @@ from typing import (
from pydantic.tools import parse_obj_as
from galaxy import exceptions
from galaxy.app import StructuredApp
from galaxy.app import MinimalManagerApp
from galaxy.files import (
ConfiguredFileSources,
ProvidesUserFileSourcesUserContext,
@@ -37,7 +37,7 @@ class RemoteFilesManager:
Interface/service object for interacting with remote files.
"""
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
self._app = app
def index(
lib/galaxy/managers/sharable.py
+2 -2
View File
@@ -34,7 +34,7 @@ from galaxy.managers import (
)
from galaxy.model import UserShareAssociation
from galaxy.schema.fields import EncodedDatabaseIdField
from galaxy.structured_app import StructuredApp
from galaxy.structured_app import MinimalManagerApp
from galaxy.util import ready_name_for_url
log = logging.getLogger(__name__)
@@ -51,7 +51,7 @@ class SharableModelManager(base.ModelManager, secured.OwnableManagerMixin, secur
#: the single character abbreviation used in username_and_slug: e.g. 'h' for histories: u/user/h/slug
SINGLE_CHAR_ABBR: Optional[str] = None
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
super().__init__(app)
# user manager is needed to check access/ownership/admin
self.user_manager = users.UserManager(app)
lib/galaxy/managers/users.py
+2 -2
View File
@@ -30,7 +30,7 @@ from galaxy.security.validate_user_input import (
validate_password,
validate_publicname
)
from galaxy.structured_app import BasicApp, StructuredApp
from galaxy.structured_app import BasicApp, MinimalManagerApp
from galaxy.util.hash_util import new_secure_hash
from galaxy.web import url_for
@@ -611,7 +611,7 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin):
class UserSerializer(base.ModelSerializer, deletable.PurgableSerializerMixin):
model_manager_class = UserManager
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
"""
Convert a User and associated data to a dictionary representation.
"""
lib/galaxy/managers/visualizations.py
+3 -3
View File
@@ -10,7 +10,7 @@ from typing import Optional
from galaxy import model
from galaxy.managers import sharable
from galaxy.schema.fields import EncodedDatabaseIdField
from galaxy.structured_app import StructuredApp
from galaxy.structured_app import MinimalManagerApp
log = logging.getLogger(__name__)
@@ -43,7 +43,7 @@ class VisualizationSerializer(sharable.SharableModelSerializer):
model_manager_class = VisualizationManager
SINGLE_CHAR_ABBR = 'v'
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
super().__init__(app)
self.visualization_manager = self.manager
@@ -82,7 +82,7 @@ class VisualizationsService:
and pydantic models to declare its parameters and return types.
"""
def __init__(self, app: StructuredApp, manager: VisualizationManager, serializer: VisualizationSerializer):
def __init__(self, app: MinimalManagerApp, manager: VisualizationManager, serializer: VisualizationSerializer):
self.app = app
self.manager = manager
self.serializer = serializer
lib/galaxy/managers/workflows.py
+3 -3
View File
@@ -26,7 +26,7 @@ from galaxy import (
)
from galaxy.jobs.actions.post import ActionBox
from galaxy.model.item_attrs import UsesAnnotations
from galaxy.structured_app import StructuredApp
from galaxy.structured_app import MinimalManagerApp
from galaxy.tools.parameters import (
params_to_incoming,
visit_input_values
@@ -69,7 +69,7 @@ class WorkflowsManager:
the galaxy.workflow module.
"""
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
self.app = app
def get_stored_workflow(self, trans, workflow_id, by_stored_id=True):
@@ -306,7 +306,7 @@ CreatedWorkflow = namedtuple("CreatedWorkflow", ["stored_workflow", "workflow",
class WorkflowContentsManager(UsesAnnotations):
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
self.app = app
self._resource_mapper_function = get_resource_mapper_function(app)
lib/galaxy/model/orm/engine_factory.py
+2
View File
@@ -3,6 +3,7 @@ import logging
import os
import threading
import time
from multiprocessing.util import register_after_fork
from sqlalchemy import (
create_engine,
@@ -87,4 +88,5 @@ def build_engine(url, engine_options, database_query_profiling_proxy=False, trac
# Create the database engine
engine = create_engine(url, **engine_options)
register_after_fork(engine, lambda e: e.dispose())
return engine
lib/galaxy/structured_app.py
+37 -5
View File
@@ -41,7 +41,43 @@ class BasicApp(Container):
datatypes_registry: Registry
class StructuredApp(BasicApp):
class MinimalApp(BasicApp):
is_webapp: bool # is_webapp will be set to true when building WSGI app
new_installation: bool
tag_handler: GalaxyTagHandler
model: GalaxyModelMapping
install_model: ModelMapping
security_agent: GalaxyRBACAgent
host_security_agent: HostAgent
class MinimalManagerApp(MinimalApp):
is_webapp: bool # is_webapp will be set to true when building WSGI app
new_installation: bool
tag_handler: GalaxyTagHandler
file_sources: ConfiguredFileSources
genome_builds: GenomeBuilds
model: GalaxyModelMapping
install_model: ModelMapping
security_agent: GalaxyRBACAgent
host_security_agent: HostAgent
dataset_collections_service: Any # 'galaxy.managers.collections.DatasetCollectionManager'
history_manager: Any # 'galaxy.managers.histories.HistoryManager'
hda_manager: Any # 'galaxy.managers.hdas.HDAManager'
workflow_manager: Any # 'galaxy.managers.workflows.WorkflowsManager'
workflow_contents_manager: Any # 'galaxy.managers.workflows.WorkflowContentsManager'
library_folder_manager: Any # 'galaxy.managers.folders.FolderManager'
library_manager: Any # 'galaxy.managers.libraries.LibraryManager'
role_manager: Any # 'galaxy.managers.roles.RoleManager'
installed_repository_manager: Any # 'galaxy.tool_shed.galaxy_install.installed_repository_manager.InstalledRepositoryManager'
user_manager: Any
@property
def is_job_handler(self) -> bool:
pass
class StructuredApp(MinimalManagerApp):
"""Interface defining typed description of the Galaxy UniverseApplication.
Ideally nothing that depends on StructuredApp should require
@@ -92,7 +128,3 @@ class StructuredApp(BasicApp):
job_manager: Any # galaxy.jobs.manager.JobManager
user_manager: Any
api_keys_manager: Any
@property
def is_job_handler(self) -> bool:
pass
lib/galaxy/tools/cache.py
+2 -2
View File
@@ -16,7 +16,7 @@ from sqlalchemy.orm import (
from sqlitedict import SqliteDict
from galaxy.model.tool_shed_install import ToolShedRepository
from galaxy.structured_app import StructuredApp
from galaxy.structured_app import MinimalManagerApp
from galaxy.tools.toolbox.base import ToolConfRepository
from galaxy.util import unicodify
from galaxy.util.hash_util import md5_hash_file
@@ -286,7 +286,7 @@ class ToolShedRepositoryCache:
repositories: List[ToolShedRepository]
repos_by_tuple: Dict[Tuple[str, str, str], List[ToolConfRepository]]
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
self.app = app
# Contains ToolConfRepository objects created from shed_tool_conf.xml entries
self.local_repositories = []
lib/galaxy/tools/data_manager/manager.py
+2 -2
View File
@@ -5,7 +5,7 @@ import os
from typing import Dict
from galaxy import util
from galaxy.structured_app import StructuredApp
from galaxy.structured_app import MinimalManagerApp
from galaxy.tools.data import TabularToolDataTable
from galaxy.util.template import fill_template
@@ -20,7 +20,7 @@ class DataManagers:
data_managers: Dict[str, 'DataManager']
managed_data_tables: Dict[str, 'DataManager']
def __init__(self, app: StructuredApp, xml_filename=None):
def __init__(self, app: MinimalManagerApp, xml_filename=None):
self.app = app
self.data_managers = {}
self.managed_data_tables = {}
lib/galaxy/visualization/genomes.py
+2 -2
View File
@@ -11,7 +11,7 @@ from galaxy.exceptions import (
ObjectNotFound,
ReferenceDataError,
)
from galaxy.structured_app import StructuredApp
from galaxy.structured_app import MinimalManagerApp
from galaxy.util.bunch import Bunch
log = logging.getLogger(__name__)
@@ -197,7 +197,7 @@ class Genomes:
Provides information about available genome data and methods for manipulating that data.
"""
def __init__(self, app: StructuredApp):
def __init__(self, app: MinimalManagerApp):
self.app = app
# Create list of genomes from app.genome_builds
self.genomes: Dict[str, Genome] = {}
lib/galaxy/webapps/galaxy/config_schema.yml
+9
View File
@@ -3183,6 +3183,15 @@ mapping:
will automatically create and use a separate sqlite database located in your
<galaxy>/database folder (indicated in the commented out line below).
enable_celery_tasks:
type: bool
default: false
required: false
desc: |
Offload long-running tasks to a Celery task queue.
Activate this only if you have setup a Celery worker for Galaxy.
For details, see https://docs.galaxyproject.org/en/master/admin/production.html
use_pbkdf2:
type: bool
default: true
lib/galaxy/webapps/galaxy/controllers/user.py
+9 -5
View File
@@ -222,11 +222,15 @@ class User(BaseUIController, UsesFormDefinitionsMixin, CreatesApiKeysMixin):
if message:
return self.message_exception(trans, message)
if trans.user:
# Queue a quota recalculation (async) task -- this takes a
# while sometimes, so we don't want to block on logout.
send_local_control_task(trans.app,
"recalculate_user_disk_usage",
kwargs={"user_id": trans.security.encode_id(trans.user.id)})
if trans.app.config.enable_celery_tasks:
# Queue a quota recalculation (async) task -- this takes a
# while sometimes, so we don't want to block on logout.
from galaxy.celery.tasks import recalculate_user_disk_usage
recalculate_user_disk_usage.delay(user_id=trans.user.id)
else:
send_local_control_task(trans.app,
"recalculate_user_disk_usage",
kwargs={"user_id": trans.security.encode_id(trans.user.id)})
# Since logging an event requires a session, we'll log prior to ending the session
trans.log_event("User logged out")
trans.handle_user_logout(logout_all=logout_all)
packages/app/galaxy/celery
Symlink
+1
View File
@@ -0,0 +1 @@
../../../lib/galaxy/celery
packages/app/requirements.txt
+1
View File
@@ -5,6 +5,7 @@ galaxy-tool-util
galaxy-web-framework
galaxy-web-stack
celery
kombu
Beaker
pykwalify
pyproject.toml
+2
View File
@@ -25,10 +25,12 @@ bleach = "*"
boltons = "*"
boto = "*"
bx-python = "*"
celery = "*"
Cheetah3 = "*"
cloudauthz = "==0.6.0"
cloudbridge = "*"
contextvars = {version = "*", python = "~3.6"}
circus = "*"
cwltool = "==3.0.20201109103151"
dictobj = "*"
docutils = "*"
test/unit/managers/test_HDAManager.py
+1 -1
View File
@@ -175,7 +175,7 @@ class HDAManagerTestCase(HDATestCase):
self.log("should purge an hda if config does allow")
self.assertFalse(item1.purged)
self.assertEqual(self.hda_manager.purge(item1), item1)
self.hda_manager.purge(item1)
self.assertTrue(item1.deleted)
self.assertTrue(item1.purged)
test/unit/unittest_utils/galaxy_mock.py
+3 -1
View File
@@ -21,7 +21,7 @@ from galaxy.model import mapping, tags
from galaxy.model.base import SharedModelMapping
from galaxy.model.mapping import GalaxyModelMapping
from galaxy.security import idencoding
from galaxy.structured_app import BasicApp, StructuredApp
from galaxy.structured_app import BasicApp, MinimalManagerApp, StructuredApp
from galaxy.tool_util.deps.containers import NullContainerFinder
from galaxy.util import StructuredExecutionTimer
from galaxy.util.bunch import Bunch
@@ -67,6 +67,7 @@ class MockApp(di.Container):
def __init__(self, config=None, **kwargs):
super().__init__()
self[BasicApp] = self
self[MinimalManagerApp] = self
self[StructuredApp] = self
self.config = config or MockAppConfig(**kwargs)
self.security = self.config.security
@@ -139,6 +140,7 @@ class MockAppConfig(Bunch):
self.security = idencoding.IdEncodingHelper(id_secret='6e46ed6483a833c100e68cc3f1d0dd76')
self.database_connection = kwargs.get('database_connection', "sqlite:///:memory:")
self.use_remote_user = kwargs.get('use_remote_user', False)
self.enable_celery_tasks = False
self.data_dir = os.path.join(root, 'database')
self.file_path = os.path.join(self.data_dir, 'files')
self.jobs_directory = os.path.join(self.data_dir, 'jobs_directory')