From 323e21f5a87bc2b9c6a96d18927da4c4e8bea330 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 23 Mar 2021 11:09:03 -0400 Subject: [PATCH 01/36] Initial circus addition -- dep update needs to be run still --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 96bfbba2693..96f9b4ceba9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ Cheetah3 = "*" cloudauthz = "==0.6.0" cloudbridge = "*" contextvars = {version = "*", python = "~3.6"} +circus = "*" cwltool = "==3.0.20201109103151" dictobj = "*" docutils = "*" From 2ce193821a08f8e30105be59ed134a5f089f125f Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 23 Mar 2021 11:09:03 -0400 Subject: [PATCH 02/36] Basic dev reloading w/ circus --- config/dev.ini | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 config/dev.ini diff --git a/config/dev.ini b/config/dev.ini new file mode 100644 index 00000000000..170db683e12 --- /dev/null +++ b/config/dev.ini @@ -0,0 +1,34 @@ +[circus] +#statsd = 1 +#httpd = 1 +debug = True + +[env] +GALAXY_CONFIG_FILE=config/galaxy.yml + +[watcher:web] +cmd = uvicorn --factory 'galaxy.webapps.galaxy.fast_factory:factory' --reload --reload-dir lib --app-dir lib --fd $(circus.sockets.web) +use_sockets = True +numprocesses = 1 +stop_signal = TERM +stop_children = True +copy_env = True +virtualenv = .venv + +[watcher:client] +working_dir = client +cmd = yarn watch +numprocesses = 1 +singleton = True +copy_env = True +stop_signal = TERM +stop_children = True + +[socket:web] +host = 0.0.0.0 +port = 8080 + +#[watcher:celery] +#copy_env = True +#cmd = celery +#args = worker --app=your_app --loglevel=DEBUG -E From f11bacf9d319ddb5411216cf9cbbbf3540f8af0a Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 23 Mar 2021 12:53:00 -0400 Subject: [PATCH 03/36] Starts a celery worker (that does nothing yet, but it does connect to my local redis) --- config/dev.ini | 14 +++++++------- lib/galaxy/tasks/__init__.py | 20 ++++++++++++++++++++ 2 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 lib/galaxy/tasks/__init__.py diff --git a/config/dev.ini b/config/dev.ini index 170db683e12..dbac682badc 100644 --- a/config/dev.ini +++ b/config/dev.ini @@ -1,6 +1,4 @@ [circus] -#statsd = 1 -#httpd = 1 debug = True [env] @@ -8,8 +6,8 @@ GALAXY_CONFIG_FILE=config/galaxy.yml [watcher:web] cmd = uvicorn --factory 'galaxy.webapps.galaxy.fast_factory:factory' --reload --reload-dir lib --app-dir lib --fd $(circus.sockets.web) -use_sockets = True numprocesses = 1 +use_sockets = True stop_signal = TERM stop_children = True copy_env = True @@ -28,7 +26,9 @@ stop_children = True host = 0.0.0.0 port = 8080 -#[watcher:celery] -#copy_env = True -#cmd = celery -#args = worker --app=your_app --loglevel=DEBUG -E +[watcher:celery] +working_dir = lib +cmd = celery +args = --app galaxy.tasks worker +copy_env = True +numprocesses = 1 diff --git a/lib/galaxy/tasks/__init__.py b/lib/galaxy/tasks/__init__.py new file mode 100644 index 00000000000..e267079fca3 --- /dev/null +++ b/lib/galaxy/tasks/__init__.py @@ -0,0 +1,20 @@ +from celery import Celery +from galaxy.util.custom_logging import get_logger +log = get_logger(__name__) + +# Test redis server, TODO: import the real app and use the same broker from config. + +app = Celery('tasks', broker='redis://localhost') + + +@app.task +def recalculate_user_disk_usage(user_id=None): + sa_session = app.model.context + if user_id: + user = sa_session.query(app.model.User).get(app.security.decode_id(user_id)) + if user: + user.calculate_and_set_disk_usage() + else: + log.error("Recalculate user disk usage task failed, user %s not found" % user_id) + else: + log.error("Recalculate user disk usage task received without user_id.") From 1c6198c42d99582e2170d13cf084892678608b69 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 23 Mar 2021 13:01:17 -0400 Subject: [PATCH 04/36] Split celery worker and tasks up --- config/dev.ini | 2 +- lib/galaxy/celery/__init__.py | 9 +++++++++ lib/galaxy/{tasks/__init__.py => celery/tasks.py} | 9 +++------ 3 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 lib/galaxy/celery/__init__.py rename lib/galaxy/{tasks/__init__.py => celery/tasks.py} (71%) diff --git a/config/dev.ini b/config/dev.ini index dbac682badc..6ef2e54cf65 100644 --- a/config/dev.ini +++ b/config/dev.ini @@ -29,6 +29,6 @@ port = 8080 [watcher:celery] working_dir = lib cmd = celery -args = --app galaxy.tasks worker +args = --app galaxy.celery worker copy_env = True numprocesses = 1 diff --git a/lib/galaxy/celery/__init__.py b/lib/galaxy/celery/__init__.py new file mode 100644 index 00000000000..1c07feba7ec --- /dev/null +++ b/lib/galaxy/celery/__init__.py @@ -0,0 +1,9 @@ +from celery import Celery + +# Test redis server, TODO: import the real app and use the same broker from config. + +app = Celery('galaxy', broker='redis://localhost', include=['galaxy.celery.tasks']) + + +if __name__ == '__main__': + app.start() diff --git a/lib/galaxy/tasks/__init__.py b/lib/galaxy/celery/tasks.py similarity index 71% rename from lib/galaxy/tasks/__init__.py rename to lib/galaxy/celery/tasks.py index e267079fca3..0bd175e31b2 100644 --- a/lib/galaxy/tasks/__init__.py +++ b/lib/galaxy/celery/tasks.py @@ -1,15 +1,12 @@ -from celery import Celery +from galaxy.celery import app from galaxy.util.custom_logging import get_logger + log = get_logger(__name__) -# Test redis server, TODO: import the real app and use the same broker from config. - -app = Celery('tasks', broker='redis://localhost') - @app.task def recalculate_user_disk_usage(user_id=None): - sa_session = app.model.context + sa_session = app.model.context # TODO: not remotely correct; this needs the actual model from our webless application context. if user_id: user = sa_session.query(app.model.User).get(app.security.decode_id(user_id)) if user: From 1174dac2962ef4902653d36ec46c84e8877158b1 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 24 Mar 2021 00:25:53 -0400 Subject: [PATCH 05/36] Move recalculate task over. Debug mode for celery, refactor a bit (still no bueno, just trying to get end-end complete) --- config/dev.ini | 2 +- lib/galaxy/celery/__init__.py | 15 ++++++++++++--- lib/galaxy/celery/tasks.py | 7 ++++--- lib/galaxy/queue_worker.py | 14 -------------- lib/galaxy/webapps/galaxy/controllers/user.py | 5 ++--- 5 files changed, 19 insertions(+), 24 deletions(-) diff --git a/config/dev.ini b/config/dev.ini index 6ef2e54cf65..be2b3aa3bcc 100644 --- a/config/dev.ini +++ b/config/dev.ini @@ -29,6 +29,6 @@ port = 8080 [watcher:celery] working_dir = lib cmd = celery -args = --app galaxy.celery worker +args = --app galaxy.celery worker -l debug copy_env = True numprocesses = 1 diff --git a/lib/galaxy/celery/__init__.py b/lib/galaxy/celery/__init__.py index 1c07feba7ec..8339930f18b 100644 --- a/lib/galaxy/celery/__init__.py +++ b/lib/galaxy/celery/__init__.py @@ -1,9 +1,18 @@ +import os from celery import Celery -# Test redis server, TODO: import the real app and use the same broker from config. +from galaxy.util.custom_logging import get_logger -app = Celery('galaxy', broker='redis://localhost', include=['galaxy.celery.tasks']) +log = get_logger(__name__) + +# This is a complete hack for now, todo: configure celery dynamically with the same galaxy internal amqp stuff. +celery_app = Celery('galaxy', broker="redis://localhost", include=['galaxy.celery.tasks']) if __name__ == '__main__': - app.start() + import galaxy.app + config_file = os.path.abspath(os.environ["GALAXY_CONFIG_FILE"]) + galaxy_app = galaxy.app.UniverseApplication(config_file=config_file) + galaxy.app.app = galaxy_app + celery_app.conf.update(broker=galaxy_app.config.amqp_internal_connection) + celery_app.start() diff --git a/lib/galaxy/celery/tasks.py b/lib/galaxy/celery/tasks.py index 0bd175e31b2..71e4d9c1785 100644 --- a/lib/galaxy/celery/tasks.py +++ b/lib/galaxy/celery/tasks.py @@ -1,12 +1,13 @@ -from galaxy.celery import app +from galaxy.celery import celery_app from galaxy.util.custom_logging import get_logger log = get_logger(__name__) -@app.task +@celery_app.task def recalculate_user_disk_usage(user_id=None): - sa_session = app.model.context # TODO: not remotely correct; this needs the actual model from our webless application context. + from galaxy.app import app + sa_session = app.model.context if user_id: user = sa_session.query(app.model.User).get(app.security.decode_id(user_id)) if user: diff --git a/lib/galaxy/queue_worker.py b/lib/galaxy/queue_worker.py index d13b1666c50..53ee5376b66 100644 --- a/lib/galaxy/queue_worker.py +++ b/lib/galaxy/queue_worker.py @@ -224,19 +224,6 @@ def reload_sanitize_allowlist(app): app.config.reload_sanitize_allowlist() -def recalculate_user_disk_usage(app, **kwargs): - user_id = kwargs.get('user_id', None) - sa_session = app.model.context - if user_id: - user = sa_session.query(app.model.User).get(app.security.decode_id(user_id)) - if user: - user.calculate_and_set_disk_usage() - else: - log.error("Recalculate user disk usage task failed, user %s not found" % user_id) - else: - log.error("Recalculate user disk usage task received without user_id.") - - def reload_tool_data_tables(app, **kwargs): path = kwargs.get('path') table_name = kwargs.get('table_name') @@ -321,7 +308,6 @@ control_message_to_task = { 'reload_job_rules': reload_job_rules, 'admin_job_lock': admin_job_lock, 'reload_sanitize_allowlist': reload_sanitize_allowlist, - 'recalculate_user_disk_usage': recalculate_user_disk_usage, 'rebuild_toolbox_search_index': rebuild_toolbox_search_index, 'reconfigure_watcher': reconfigure_watcher, 'reload_tour': reload_tour, diff --git a/lib/galaxy/webapps/galaxy/controllers/user.py b/lib/galaxy/webapps/galaxy/controllers/user.py index e1749d5d790..e34860c4b23 100644 --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -224,9 +224,8 @@ class User(BaseUIController, UsesFormDefinitionsMixin, CreatesApiKeysMixin): if trans.user: # Queue a quota recalculation (async) task -- this takes a # while sometimes, so we don't want to block on logout. - send_local_control_task(trans.app, - "recalculate_user_disk_usage", - kwargs={"user_id": trans.security.encode_id(trans.user.id)}) + from galaxy.celery.tasks import recalculate_user_disk_usage + recalculate_user_disk_usage.delay(user_id= trans.security.encode_id(trans.user.id)) # Since logging an event requires a session, we'll log prior to ending the session trans.log_event("User logged out") trans.handle_user_logout(logout_all=logout_all) From 18cbfac7c9eba1870eb7e22d27daf27b1e309376 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 24 Mar 2021 01:02:16 -0400 Subject: [PATCH 06/36] rebase all this out, but try to get galaxy's config loading correctly in this small context --- lib/galaxy/celery/__init__.py | 9 ++++----- lib/galaxy/celery/tasks.py | 11 ++++++++++- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/celery/__init__.py b/lib/galaxy/celery/__init__.py index 8339930f18b..5aa8d6f9e14 100644 --- a/lib/galaxy/celery/__init__.py +++ b/lib/galaxy/celery/__init__.py @@ -10,9 +10,8 @@ celery_app = Celery('galaxy', broker="redis://localhost", include=['galaxy.celer if __name__ == '__main__': - import galaxy.app - config_file = os.path.abspath(os.environ["GALAXY_CONFIG_FILE"]) - galaxy_app = galaxy.app.UniverseApplication(config_file=config_file) - galaxy.app.app = galaxy_app - celery_app.conf.update(broker=galaxy_app.config.amqp_internal_connection) + #import galaxy.app + #config_file = os.path.abspath(os.environ["GALAXY_CONFIG_FILE"]) + #galaxy_app = galaxy.app.UniverseApplication(config_file=config_file) + #celery_app.conf.update(broker=galaxy_app.config.amqp_internal_connection) celery_app.start() diff --git a/lib/galaxy/celery/tasks.py b/lib/galaxy/celery/tasks.py index 71e4d9c1785..efba56e0667 100644 --- a/lib/galaxy/celery/tasks.py +++ b/lib/galaxy/celery/tasks.py @@ -1,12 +1,21 @@ +import os from galaxy.celery import celery_app from galaxy.util.custom_logging import get_logger log = get_logger(__name__) +def get_galaxy_app(): + import galaxy.app + config_file = os.path.abspath(os.environ["GALAXY_CONFIG_FILE"]) + log.debug(f"config_file is {config_file}") + return galaxy.app.UniverseApplication(global_conf={'__file__':config_file}) + + @celery_app.task def recalculate_user_disk_usage(user_id=None): - from galaxy.app import app + # This is not initializing like I want it to I don't think. + app = get_galaxy_app() sa_session = app.model.context if user_id: user = sa_session.query(app.model.User).get(app.security.decode_id(user_id)) From 8702980d40495cd3d57f52670091ccc4056d5117 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 24 Mar 2021 01:11:52 -0400 Subject: [PATCH 07/36] Swap to gunicorn w/ uvicorn worker --- config/dev.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/dev.ini b/config/dev.ini index be2b3aa3bcc..41b0386b1f9 100644 --- a/config/dev.ini +++ b/config/dev.ini @@ -5,7 +5,8 @@ debug = True GALAXY_CONFIG_FILE=config/galaxy.yml [watcher:web] -cmd = uvicorn --factory 'galaxy.webapps.galaxy.fast_factory:factory' --reload --reload-dir lib --app-dir lib --fd $(circus.sockets.web) +cmd = gunicorn 'galaxy.webapps.galaxy.fast_factory:factory()' --pythonpath lib -k uvicorn.workers.UvicornWorker -b fd://$(circus.sockets.web) +send_hup = true numprocesses = 1 use_sockets = True stop_signal = TERM From 3967906260c53da26c40d868e312b4dedd95e560 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 24 Mar 2021 08:01:37 -0400 Subject: [PATCH 08/36] Incremental config changes; this makes schema loader resolve correctly --- config/dev.ini | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/dev.ini b/config/dev.ini index 41b0386b1f9..1bf93b5a6ba 100644 --- a/config/dev.ini +++ b/config/dev.ini @@ -3,6 +3,7 @@ debug = True [env] GALAXY_CONFIG_FILE=config/galaxy.yml +PYTHONPATH=lib [watcher:web] cmd = gunicorn 'galaxy.webapps.galaxy.fast_factory:factory()' --pythonpath lib -k uvicorn.workers.UvicornWorker -b fd://$(circus.sockets.web) @@ -12,7 +13,6 @@ use_sockets = True stop_signal = TERM stop_children = True copy_env = True -virtualenv = .venv [watcher:client] working_dir = client @@ -28,7 +28,6 @@ host = 0.0.0.0 port = 8080 [watcher:celery] -working_dir = lib cmd = celery args = --app galaxy.celery worker -l debug copy_env = True From 9f143cfef007dd1ada8f2b1716a86d496e52a0ad Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 24 Mar 2021 08:31:18 -0400 Subject: [PATCH 09/36] Sync up get_galaxy_app; seems the same functionally --- lib/galaxy/celery/__init__.py | 20 ++++++++++++++++---- lib/galaxy/celery/tasks.py | 10 +--------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/celery/__init__.py b/lib/galaxy/celery/__init__.py index 5aa8d6f9e14..20500b39aa2 100644 --- a/lib/galaxy/celery/__init__.py +++ b/lib/galaxy/celery/__init__.py @@ -1,5 +1,6 @@ import os from celery import Celery +from galaxy.util.properties import load_app_properties from galaxy.util.custom_logging import get_logger @@ -9,9 +10,20 @@ log = get_logger(__name__) celery_app = Celery('galaxy', broker="redis://localhost", include=['galaxy.celery.tasks']) +def get_galaxy_app(): + import galaxy.app + if galaxy.app.app: + return galaxy.app.app + config_file = os.path.abspath(os.environ["GALAXY_CONFIG_FILE"]) + kwargs = load_app_properties( + config_file=config_file, + config_section='galaxy', + ) + galaxy_app = galaxy.app.UniverseApplication(**kwargs) + return galaxy_app + + if __name__ == '__main__': - #import galaxy.app - #config_file = os.path.abspath(os.environ["GALAXY_CONFIG_FILE"]) - #galaxy_app = galaxy.app.UniverseApplication(config_file=config_file) - #celery_app.conf.update(broker=galaxy_app.config.amqp_internal_connection) + galaxy_app = get_galaxy_app() + celery_app.conf.update(broker=galaxy_app.config.amqp_internal_connection) celery_app.start() diff --git a/lib/galaxy/celery/tasks.py b/lib/galaxy/celery/tasks.py index efba56e0667..2881586a064 100644 --- a/lib/galaxy/celery/tasks.py +++ b/lib/galaxy/celery/tasks.py @@ -1,17 +1,9 @@ -import os -from galaxy.celery import celery_app +from galaxy.celery import celery_app, get_galaxy_app from galaxy.util.custom_logging import get_logger log = get_logger(__name__) -def get_galaxy_app(): - import galaxy.app - config_file = os.path.abspath(os.environ["GALAXY_CONFIG_FILE"]) - log.debug(f"config_file is {config_file}") - return galaxy.app.UniverseApplication(global_conf={'__file__':config_file}) - - @celery_app.task def recalculate_user_disk_usage(user_id=None): # This is not initializing like I want it to I don't think. From 50588ef2da0448c37f7dc8e7ca592c85de2a3882 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 24 Mar 2021 09:12:45 -0400 Subject: [PATCH 10/36] It actually works and does a recalc with this stripped down context -- we need a cleaner way to get this --- lib/galaxy/celery/__init__.py | 4 +- lib/galaxy/celery/tasks.py | 40 +++++++++++++++++-- lib/galaxy/webapps/galaxy/controllers/user.py | 5 +-- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/celery/__init__.py b/lib/galaxy/celery/__init__.py index 20500b39aa2..8fdb6f9b143 100644 --- a/lib/galaxy/celery/__init__.py +++ b/lib/galaxy/celery/__init__.py @@ -24,6 +24,6 @@ def get_galaxy_app(): if __name__ == '__main__': - galaxy_app = get_galaxy_app() - celery_app.conf.update(broker=galaxy_app.config.amqp_internal_connection) + #galaxy_app = get_galaxy_app() + #celery_app.conf.update(broker=galaxy_app.config.amqp_internal_connection) celery_app.start() diff --git a/lib/galaxy/celery/tasks.py b/lib/galaxy/celery/tasks.py index 2881586a064..b149523c221 100644 --- a/lib/galaxy/celery/tasks.py +++ b/lib/galaxy/celery/tasks.py @@ -1,16 +1,50 @@ from galaxy.celery import celery_app, get_galaxy_app from galaxy.util.custom_logging import get_logger +import datetime +import decimal +import os.path +import sys + +# Setup DB scripting environment +from sqlalchemy import * # noqa +from sqlalchemy.orm import * # noqa +from sqlalchemy.exc import * # noqa +from sqlalchemy.sql import label # noqa + +from galaxy.datatypes.registry import Registry +from galaxy.model import * # noqa + log = get_logger(__name__) +def get_galaxy_context(): + # This insanity is because get_config (I think) is not cooperating with celery arg handling + _preserved_argv = list(sys.argv[1:]) + del sys.argv[1:] + + from galaxy.model import set_datatypes_registry + from galaxy.model.mapping import init + from galaxy.model.orm.scripts import get_config + + registry = Registry() + registry.load_datatypes() + set_datatypes_registry(registry) + db_url = get_config(sys.argv)['db_url'] + sa_session = init('/tmp/', db_url).context + + # Put the args back + sys.argv.extend(_preserved_argv) + return sa_session + + @celery_app.task def recalculate_user_disk_usage(user_id=None): # This is not initializing like I want it to I don't think. - app = get_galaxy_app() - sa_session = app.model.context + sa_session = get_galaxy_context() + from galaxy import model if user_id: - user = sa_session.query(app.model.User).get(app.security.decode_id(user_id)) + user = sa_session.query(model.User).get(user_id) if user: user.calculate_and_set_disk_usage() else: diff --git a/lib/galaxy/webapps/galaxy/controllers/user.py b/lib/galaxy/webapps/galaxy/controllers/user.py index e34860c4b23..29bba4aa2f2 100644 --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -13,9 +13,9 @@ from galaxy import ( util, web ) +from galaxy.celery.tasks import recalculate_user_disk_usage from galaxy.exceptions import Conflict from galaxy.managers import users -from galaxy.queue_worker import send_local_control_task from galaxy.security.validate_user_input import ( validate_email, validate_publicname @@ -224,8 +224,7 @@ class User(BaseUIController, UsesFormDefinitionsMixin, CreatesApiKeysMixin): if trans.user: # Queue a quota recalculation (async) task -- this takes a # while sometimes, so we don't want to block on logout. - from galaxy.celery.tasks import recalculate_user_disk_usage - recalculate_user_disk_usage.delay(user_id= trans.security.encode_id(trans.user.id)) + recalculate_user_disk_usage.delay(user_id=trans.user.id) # Since logging an event requires a session, we'll log prior to ending the session trans.log_event("User logged out") trans.handle_user_logout(logout_all=logout_all) From bab45f11498ebaaa4f4140e59f139ceebbc948fe Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 24 Mar 2021 09:35:32 -0400 Subject: [PATCH 11/36] Trim down and clean up required context, add a log message to recalc task (it works!) --- lib/galaxy/celery/tasks.py | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/lib/galaxy/celery/tasks.py b/lib/galaxy/celery/tasks.py index b149523c221..ae266484afb 100644 --- a/lib/galaxy/celery/tasks.py +++ b/lib/galaxy/celery/tasks.py @@ -1,19 +1,11 @@ -from galaxy.celery import celery_app, get_galaxy_app -from galaxy.util.custom_logging import get_logger - -import datetime -import decimal -import os.path import sys -# Setup DB scripting environment -from sqlalchemy import * # noqa -from sqlalchemy.orm import * # noqa -from sqlalchemy.exc import * # noqa -from sqlalchemy.sql import label # noqa - +from galaxy.celery import celery_app from galaxy.datatypes.registry import Registry -from galaxy.model import * # noqa +from galaxy.model import (set_datatypes_registry, User) +from galaxy.model.mapping import init +from galaxy.model.orm.scripts import get_config +from galaxy.util.custom_logging import get_logger log = get_logger(__name__) @@ -23,10 +15,6 @@ def get_galaxy_context(): _preserved_argv = list(sys.argv[1:]) del sys.argv[1:] - from galaxy.model import set_datatypes_registry - from galaxy.model.mapping import init - from galaxy.model.orm.scripts import get_config - registry = Registry() registry.load_datatypes() set_datatypes_registry(registry) @@ -40,13 +28,12 @@ def get_galaxy_context(): @celery_app.task def recalculate_user_disk_usage(user_id=None): - # This is not initializing like I want it to I don't think. sa_session = get_galaxy_context() - from galaxy import model if user_id: - user = sa_session.query(model.User).get(user_id) + user = sa_session.query(User).get(user_id) if user: user.calculate_and_set_disk_usage() + log.info(f"New user disk usage is {user.disk_usage}") else: log.error("Recalculate user disk usage task failed, user %s not found" % user_id) else: From aa473447d9ea0796b7f2fa255f51ec9e26ef9121 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 24 Mar 2021 15:20:11 +0100 Subject: [PATCH 12/36] Hack in dynamic reading of broker from galaxy.yml --- lib/galaxy/celery/__init__.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/celery/__init__.py b/lib/galaxy/celery/__init__.py index 8fdb6f9b143..3ac1f8b196c 100644 --- a/lib/galaxy/celery/__init__.py +++ b/lib/galaxy/celery/__init__.py @@ -1,29 +1,32 @@ import os from celery import Celery from galaxy.util.properties import load_app_properties - +from galaxy.config import Configuration from galaxy.util.custom_logging import get_logger log = get_logger(__name__) -# This is a complete hack for now, todo: configure celery dynamically with the same galaxy internal amqp stuff. -celery_app = Celery('galaxy', broker="redis://localhost", include=['galaxy.celery.tasks']) - def get_galaxy_app(): import galaxy.app if galaxy.app.app: return galaxy.app.app - config_file = os.path.abspath(os.environ["GALAXY_CONFIG_FILE"]) - kwargs = load_app_properties( - config_file=config_file, - config_section='galaxy', - ) - galaxy_app = galaxy.app.UniverseApplication(**kwargs) + galaxy_app = galaxy.app.UniverseApplication(**get_galaxy_config()) return galaxy_app +def get_galaxy_config(): + config_file = os.path.abspath(os.environ["GALAXY_CONFIG_FILE"]) + return load_app_properties( + config_file=config_file, + config_section='galaxy', + ) +broker = Configuration(**get_galaxy_config()).amqp_internal_connection +# This is a complete hack for now, todo: configure celery dynamically with the same galaxy internal amqp stuff. +celery_app = Celery('galaxy', broker=broker, include=['galaxy.celery.tasks']) +log.warning(f"BROKER IS {broker}") + + if __name__ == '__main__': - #galaxy_app = get_galaxy_app() #celery_app.conf.update(broker=galaxy_app.config.amqp_internal_connection) celery_app.start() From dc79beafb3e55bdeb7fa5ced8829b5364f2b3b67 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 24 Mar 2021 16:45:44 -0400 Subject: [PATCH 13/36] Add circus; update all --- lib/galaxy/dependencies/dev-requirements.txt | 9 ++++++--- lib/galaxy/dependencies/pinned-requirements.txt | 8 ++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index 82e26ec3016..a7b5ac2fa6f 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -28,9 +28,10 @@ cachecontrol==0.11.7; python_version >= "3.6" and python_version < "4" cached-property==1.5.2; python_version < "3.8" and python_version >= "3.6" cachetools==4.2.1; python_version >= "3.5" and python_version < "4.0" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") certifi==2020.12.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6" -cffi==1.14.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +cffi==1.14.5; implementation_name == "pypy" and python_version >= "3.6" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0") chardet==4.0.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" cheetah3==3.2.6.post1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") +circus==0.17.1 click==7.1.2; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" cliff==3.7.0; python_version >= "3.6" cloudauthz==0.6.0 @@ -136,10 +137,10 @@ protobuf==3.15.6; python_version >= "3.6" and python_full_version < "3.0.0" or p prov==1.5.1; python_version >= "3.6" and python_version < "4" psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") pulsar-galaxy-lib==0.14.2 -py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or python_full_version >= "3.4.0" and python_version >= "3.6" and implementation_name == "pypy" pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" pyasn1==0.4.8; python_version >= "3.5" and python_version < "4" -pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.6" and python_full_version >= "3.4.0" pycryptodome==3.10.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") pydantic==1.7.3; python_version >= "3.6" and python_version < "4.0" pydot==1.4.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.4.0" @@ -178,6 +179,7 @@ python3-openid==3.2.0; python_version >= "3.0" pytz==2021.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" pyuwsgi==2.0.19.1.post0 pyyaml==5.4.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") +pyzmq==22.0.3; python_version >= "3.6" rdflib-jsonld==0.5.0; python_version >= "3.6" and python_version < "4" rdflib==4.2.2; python_version >= "3.6" and python_version < "4" recommonmark==0.7.1 @@ -226,6 +228,7 @@ tempita==0.5.2 tenacity==7.0.0 testfixtures==6.17.1 toml==0.10.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" +tornado==6.1; python_version >= "3.5" tqdm==4.59.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" twill==3.0 typing-extensions==3.7.4.3; python_version >= "3.6" and python_version < "3.8" diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index c4b41ed44ed..6489ec807c5 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -26,9 +26,10 @@ cachecontrol==0.11.7; python_version >= "3.6" and python_version < "4" cached-property==1.5.2; python_version < "3.8" and python_version >= "3.6" cachetools==4.2.1; python_version >= "3.5" and python_version < "4.0" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") certifi==2020.12.5; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" -cffi==1.14.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +cffi==1.14.5; implementation_name == "pypy" and python_version >= "3.6" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0") chardet==4.0.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" cheetah3==3.2.6.post1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") +circus==0.17.1 click==7.1.2; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" cliff==3.7.0; python_version >= "3.6" cloudauthz==0.6.0 @@ -119,9 +120,10 @@ protobuf==3.15.6; python_version >= "3.6" and python_full_version < "3.0.0" or p prov==1.5.1; python_version >= "3.6" and python_version < "4" psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") pulsar-galaxy-lib==0.14.2 +py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.6" and python_full_version >= "3.4.0" pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" pyasn1==0.4.8; python_version >= "3.5" and python_version < "4" -pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.6" and python_full_version >= "3.4.0" pycryptodome==3.10.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") pydantic==1.7.3; python_version >= "3.6" and python_version < "4.0" pydot==1.4.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.4.0" @@ -147,6 +149,7 @@ python3-openid==3.2.0; python_version >= "3.0" pytz==2021.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" pyuwsgi==2.0.19.1.post0 pyyaml==5.4.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") +pyzmq==22.0.3; python_version >= "3.6" rdflib-jsonld==0.5.0; python_version >= "3.6" and python_version < "4" rdflib==4.2.2; python_version >= "3.6" and python_version < "4" refgenconf==0.9.3 @@ -178,6 +181,7 @@ stevedore==3.3.0; python_version >= "3.6" svgwrite==1.4.1; python_version >= "3.6" tempita==0.5.2 tenacity==7.0.0 +tornado==6.1; python_version >= "3.5" tqdm==4.59.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" typing-extensions==3.7.4.3; python_version >= "3.6" and python_version < "3.8" tzlocal==2.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" From 8db40adf8a63a57bcbf6db8ebb42fc4a447e1340 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 25 Mar 2021 13:23:24 +0100 Subject: [PATCH 14/36] Add celery --- lib/galaxy/dependencies/dev-requirements.txt | 8 +++++++- lib/galaxy/dependencies/pinned-requirements.txt | 8 +++++++- pyproject.toml | 1 + 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index a7b5ac2fa6f..55047974fff 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -17,6 +17,7 @@ bagit==1.8.1; python_version >= "3.6" and python_full_version < "3.0.0" and pyth bcrypt==3.2.0; python_version >= "3.6" bdbag==1.6.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0" and python_version < "4") beaker==1.11.0 +billiard==3.6.3.0; python_version >= "3.6" bioblend==0.15.0; python_version >= "3.6" bleach==3.3.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") boltons==20.2.1 @@ -27,12 +28,16 @@ bx-python==0.8.11; python_version >= "3.6" cachecontrol==0.11.7; python_version >= "3.6" and python_version < "4" cached-property==1.5.2; python_version < "3.8" and python_version >= "3.6" cachetools==4.2.1; python_version >= "3.5" and python_version < "4.0" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") +celery==5.0.5; python_version >= "3.6" certifi==2020.12.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6" cffi==1.14.5; implementation_name == "pypy" and python_version >= "3.6" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0") chardet==4.0.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" cheetah3==3.2.6.post1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") circus==0.17.1 -click==7.1.2; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" +click-didyoumean==0.0.3; python_version >= "3.6" +click-plugins==1.1.1; python_version >= "3.6" +click-repl==0.1.6; python_version >= "3.6" +click==7.1.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" cliff==3.7.0; python_version >= "3.6" cloudauthz==0.6.0 cloudbridge==2.1.0 @@ -133,6 +138,7 @@ pbr==5.5.1; python_version >= "3.6" pluggy==0.13.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" port-for==0.4; python_version >= "3.6" prettytable==0.7.2; python_version >= "3.6" +prompt-toolkit==3.0.3; python_version >= "3.6" protobuf==3.15.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" prov==1.5.1; python_version >= "3.6" and python_version < "4" psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 6489ec807c5..f52b8054d07 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -15,6 +15,7 @@ bagit==1.8.1; python_version >= "3.6" and python_full_version < "3.0.0" and pyth bcrypt==3.2.0; python_version >= "3.6" bdbag==1.6.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0" and python_version < "4") beaker==1.11.0 +billiard==3.6.3.0; python_version >= "3.6" bioblend==0.15.0; python_version >= "3.6" bleach==3.3.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") boltons==20.2.1 @@ -25,12 +26,16 @@ bx-python==0.8.11; python_version >= "3.6" cachecontrol==0.11.7; python_version >= "3.6" and python_version < "4" cached-property==1.5.2; python_version < "3.8" and python_version >= "3.6" cachetools==4.2.1; python_version >= "3.5" and python_version < "4.0" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") +celery==5.0.5; python_version >= "3.6" certifi==2020.12.5; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" cffi==1.14.5; implementation_name == "pypy" and python_version >= "3.6" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0") chardet==4.0.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" cheetah3==3.2.6.post1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") circus==0.17.1 -click==7.1.2; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" +click-didyoumean==0.0.3; python_version >= "3.6" +click-plugins==1.1.1; python_version >= "3.6" +click-repl==0.1.6; python_version >= "3.6" +click==7.1.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" cliff==3.7.0; python_version >= "3.6" cloudauthz==0.6.0 cloudbridge==2.1.0 @@ -116,6 +121,7 @@ paste==3.5.0 pastedeploy==2.1.1 pbr==5.5.1; python_version >= "3.6" prettytable==0.7.2; python_version >= "3.6" +prompt-toolkit==3.0.3; python_version >= "3.6" protobuf==3.15.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" prov==1.5.1; python_version >= "3.6" and python_version < "4" psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") diff --git a/pyproject.toml b/pyproject.toml index 96f9b4ceba9..6cb68917621 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ bleach = "*" boltons = "*" boto = "*" bx-python = "*" +celery = "*" Cheetah3 = "*" cloudauthz = "==0.6.0" cloudbridge = "*" From 426ca4f206e4e97d6e7e3281b734b600e91ad71b Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 25 Mar 2021 17:04:48 +0100 Subject: [PATCH 15/36] Split UniverseApplication --- lib/galaxy/app.py | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index eb69840d8a5..9dd5821f9e2 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -76,9 +76,7 @@ log = logging.getLogger(__name__) app = None -class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container): - """Encapsulates the state of a Universe application""" - +class InitializedApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container): def __init__(self, **kwargs) -> None: super().__init__() self._register_singleton(BasicApp, self) @@ -91,7 +89,6 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container log.debug("python path is: %s", ", ".join(sys.path)) self.name = 'galaxy' self.is_webapp = False - startup_timer = ExecutionTimer() self.new_installation = False # Read config file and check for errors self.config: Any = self._register_singleton(config.Configuration, config.Configuration(**kwargs)) @@ -103,13 +100,7 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container self.application_stack = self._register_singleton(ApplicationStack, application_stack_instance(app=self)) self.application_stack.register_postfork_function(self.application_stack.set_postfork_server_name, self) self.config.reload_sanitize_allowlist(explicit='sanitize_allowlist_file' in kwargs) - self.amqp_internal_connection_obj = galaxy.queues.connection_from_config(self.config) - # queue_worker *can* be initialized with a queue, but here we don't - # want to and we'll allow postfork to bind and start it. - self.queue_worker = self._register_singleton(GalaxyQueueWorker, GalaxyQueueWorker(self)) - - self._configure_tool_shed_registry() - self._configure_object_store(fsmon=True) + self._configure_object_store(fsmon=False) # Setup the database engine and ORM config_file = kwargs.get('global_conf', {}).get('__file__', None) if config_file: @@ -143,6 +134,27 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container # ConfiguredFileSources self.file_sources = self._register_singleton(ConfiguredFileSources, ConfiguredFileSources.from_app_config(self.config)) + # Load security policy. + self.security_agent = self.model.security_agent + self.host_security_agent = galaxy.model.security.HostAgent( + model=self.security_agent.model, + permitted_actions=self.security_agent.permitted_actions) + # Load quota management. + self.quota_agent = self._register_singleton(QuotaAgent, get_quota_agent(self.config, self.model)) + +class UniverseApplication(InitializedApplication): + """Encapsulates the state of a Universe application""" + + def __init__(self, **kwargs) -> None: + startup_timer = ExecutionTimer() + super().__init__(**kwargs) + self.amqp_internal_connection_obj = galaxy.queues.connection_from_config(self.config) + # queue_worker *can* be initialized with a queue, but here we don't + # want to and we'll allow postfork to bind and start it. + self.queue_worker = self._register_singleton(GalaxyQueueWorker, GalaxyQueueWorker(self)) + + self._configure_tool_shed_registry() + self._configure_object_store(fsmon=True) # Tool Data Tables self._configure_tool_data_tables(from_shed_config=False) @@ -203,13 +215,7 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container self[ToursRegistry] = tour_registry # type: ignore # Webhooks registry self.webhooks_registry = self._register_singleton(WebhooksRegistry, WebhooksRegistry(self.config.webhooks_dir)) - # Load security policy. - self.security_agent = self.model.security_agent - self.host_security_agent = galaxy.model.security.HostAgent( - model=self.security_agent.model, - permitted_actions=self.security_agent.permitted_actions) - # Load quota management. - self.quota_agent = self._register_singleton(QuotaAgent, get_quota_agent(self.config, self.model)) + # Heartbeat for thread profiling self.heartbeat = None self.auth_manager = self._register_singleton(auth.AuthManager, auth.AuthManager(self.config)) From 65210eb7edc75f1780340b7bc346c9fe457c0599 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 25 Mar 2021 18:00:36 +0100 Subject: [PATCH 16/36] Import MinimalGalaxyApplication --- lib/galaxy/celery/__init__.py | 12 +++++++----- lib/galaxy/celery/tasks.py | 27 ++++++--------------------- 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/lib/galaxy/celery/__init__.py b/lib/galaxy/celery/__init__.py index 3ac1f8b196c..debd75a029d 100644 --- a/lib/galaxy/celery/__init__.py +++ b/lib/galaxy/celery/__init__.py @@ -1,8 +1,10 @@ import os + from celery import Celery -from galaxy.util.properties import load_app_properties + from galaxy.config import Configuration from galaxy.util.custom_logging import get_logger +from galaxy.util.properties import load_app_properties log = get_logger(__name__) @@ -11,22 +13,22 @@ def get_galaxy_app(): import galaxy.app if galaxy.app.app: return galaxy.app.app - galaxy_app = galaxy.app.UniverseApplication(**get_galaxy_config()) + galaxy_app = galaxy.app.MinimalGalaxyApplication(**get_galaxy_config()) return galaxy_app def get_galaxy_config(): - config_file = os.path.abspath(os.environ["GALAXY_CONFIG_FILE"]) + config_file = os.path.abspath(os.environ.get("GALAXY_CONFIG_FILE")) return load_app_properties( config_file=config_file, config_section='galaxy', ) + + broker = Configuration(**get_galaxy_config()).amqp_internal_connection -# This is a complete hack for now, todo: configure celery dynamically with the same galaxy internal amqp stuff. celery_app = Celery('galaxy', broker=broker, include=['galaxy.celery.tasks']) log.warning(f"BROKER IS {broker}") if __name__ == '__main__': - #celery_app.conf.update(broker=galaxy_app.config.amqp_internal_connection) celery_app.start() diff --git a/lib/galaxy/celery/tasks.py b/lib/galaxy/celery/tasks.py index ae266484afb..e8367642a1f 100644 --- a/lib/galaxy/celery/tasks.py +++ b/lib/galaxy/celery/tasks.py @@ -1,36 +1,21 @@ -import sys - from galaxy.celery import celery_app -from galaxy.datatypes.registry import Registry -from galaxy.model import (set_datatypes_registry, User) -from galaxy.model.mapping import init -from galaxy.model.orm.scripts import get_config from galaxy.util.custom_logging import get_logger +from . import get_galaxy_app log = get_logger(__name__) def get_galaxy_context(): - # This insanity is because get_config (I think) is not cooperating with celery arg handling - _preserved_argv = list(sys.argv[1:]) - del sys.argv[1:] - - registry = Registry() - registry.load_datatypes() - set_datatypes_registry(registry) - db_url = get_config(sys.argv)['db_url'] - sa_session = init('/tmp/', db_url).context - - # Put the args back - sys.argv.extend(_preserved_argv) - return sa_session + app = get_galaxy_app() + return app @celery_app.task def recalculate_user_disk_usage(user_id=None): - sa_session = get_galaxy_context() + app = get_galaxy_context() + sa_session = app.model.session if user_id: - user = sa_session.query(User).get(user_id) + user = sa_session.query(app.model.User).get(user_id) if user: user.calculate_and_set_disk_usage() log.info(f"New user disk usage is {user.disk_usage}") From 3fd2a3ee7a31c769d50c60444b9fa7a6aeffa14c Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 25 Mar 2021 18:05:25 +0100 Subject: [PATCH 17/36] Revert "Split UniverseApplication" This reverts commit 95f5ebd7122e0301c6458a11ba91068b4336734f. --- lib/galaxy/app.py | 42 ++++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 9dd5821f9e2..eb69840d8a5 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -76,7 +76,9 @@ log = logging.getLogger(__name__) app = None -class InitializedApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container): +class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container): + """Encapsulates the state of a Universe application""" + def __init__(self, **kwargs) -> None: super().__init__() self._register_singleton(BasicApp, self) @@ -89,6 +91,7 @@ class InitializedApplication(StructuredApp, config.ConfiguresGalaxyMixin, Contai log.debug("python path is: %s", ", ".join(sys.path)) self.name = 'galaxy' self.is_webapp = False + startup_timer = ExecutionTimer() self.new_installation = False # Read config file and check for errors self.config: Any = self._register_singleton(config.Configuration, config.Configuration(**kwargs)) @@ -100,7 +103,13 @@ class InitializedApplication(StructuredApp, config.ConfiguresGalaxyMixin, Contai self.application_stack = self._register_singleton(ApplicationStack, application_stack_instance(app=self)) self.application_stack.register_postfork_function(self.application_stack.set_postfork_server_name, self) self.config.reload_sanitize_allowlist(explicit='sanitize_allowlist_file' in kwargs) - self._configure_object_store(fsmon=False) + self.amqp_internal_connection_obj = galaxy.queues.connection_from_config(self.config) + # queue_worker *can* be initialized with a queue, but here we don't + # want to and we'll allow postfork to bind and start it. + self.queue_worker = self._register_singleton(GalaxyQueueWorker, GalaxyQueueWorker(self)) + + self._configure_tool_shed_registry() + self._configure_object_store(fsmon=True) # Setup the database engine and ORM config_file = kwargs.get('global_conf', {}).get('__file__', None) if config_file: @@ -134,27 +143,6 @@ class InitializedApplication(StructuredApp, config.ConfiguresGalaxyMixin, Contai # ConfiguredFileSources self.file_sources = self._register_singleton(ConfiguredFileSources, ConfiguredFileSources.from_app_config(self.config)) - # Load security policy. - self.security_agent = self.model.security_agent - self.host_security_agent = galaxy.model.security.HostAgent( - model=self.security_agent.model, - permitted_actions=self.security_agent.permitted_actions) - # Load quota management. - self.quota_agent = self._register_singleton(QuotaAgent, get_quota_agent(self.config, self.model)) - -class UniverseApplication(InitializedApplication): - """Encapsulates the state of a Universe application""" - - def __init__(self, **kwargs) -> None: - startup_timer = ExecutionTimer() - super().__init__(**kwargs) - self.amqp_internal_connection_obj = galaxy.queues.connection_from_config(self.config) - # queue_worker *can* be initialized with a queue, but here we don't - # want to and we'll allow postfork to bind and start it. - self.queue_worker = self._register_singleton(GalaxyQueueWorker, GalaxyQueueWorker(self)) - - self._configure_tool_shed_registry() - self._configure_object_store(fsmon=True) # Tool Data Tables self._configure_tool_data_tables(from_shed_config=False) @@ -215,7 +203,13 @@ class UniverseApplication(InitializedApplication): self[ToursRegistry] = tour_registry # type: ignore # Webhooks registry self.webhooks_registry = self._register_singleton(WebhooksRegistry, WebhooksRegistry(self.config.webhooks_dir)) - + # Load security policy. + self.security_agent = self.model.security_agent + self.host_security_agent = galaxy.model.security.HostAgent( + model=self.security_agent.model, + permitted_actions=self.security_agent.permitted_actions) + # Load quota management. + self.quota_agent = self._register_singleton(QuotaAgent, get_quota_agent(self.config, self.model)) # Heartbeat for thread profiling self.heartbeat = None self.auth_manager = self._register_singleton(auth.AuthManager, auth.AuthManager(self.config)) From cac4bc161203df228021ad4101d12f26453d5843 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 25 Mar 2021 10:00:32 -0400 Subject: [PATCH 18/36] WIP: minimal app? --- lib/galaxy/app.py | 163 +++++++++++++++++++++++++--------------------- 1 file changed, 87 insertions(+), 76 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index eb69840d8a5..104b7557328 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -2,7 +2,7 @@ import logging import signal import sys import time -from typing import Any +from typing import Any, Callable, List, Tuple from sqlalchemy.orm.scoping import ( scoped_session, @@ -76,13 +76,35 @@ log = logging.getLogger(__name__) app = None -class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container): - """Encapsulates the state of a Universe application""" +class HaltableContainer(Container): + haltables: List[Tuple[str, Callable]] - def __init__(self, **kwargs) -> None: + def __init__(self) -> None: super().__init__() + self.haltables = [] + + def shutdown(self): + exception = None + for what, haltable in self.haltables: + try: + haltable() + except Exception as e: + log.exception(f"Failed to shutdown {what} cleanly") + exception = exception or e + if exception is not None: + raise exception + + +class MinimalGalaxyApplication(BasicApp, config.ConfiguresGalaxyMixin, HaltableContainer): + """Encapsulates the state of a minimal Galaxy application""" + + def __init__(self, fsmon=False, **kwargs) -> None: + super().__init__() + self.haltables = [ + ("object store", self._shutdown_object_store), + ("database connection", self._shutdown_model), + ] self._register_singleton(BasicApp, self) - self._register_singleton(StructuredApp, self) if not log.handlers: # Paste didn't handle it, so we need a temporary basic log # configured. The handler added here gets dumped and replaced with @@ -91,12 +113,50 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container log.debug("python path is: %s", ", ".join(sys.path)) self.name = 'galaxy' self.is_webapp = False - startup_timer = ExecutionTimer() self.new_installation = False # Read config file and check for errors self.config: Any = self._register_singleton(config.Configuration, config.Configuration(**kwargs)) self.config.check() config.configure_logging(self.config) + self._configure_object_store(fsmon=True) + config_file = kwargs.get('global_conf', {}).get('__file__', None) + if config_file: + log.debug('Using "galaxy.ini" config file: %s', config_file) + check_migrate_tools = self.config.check_migrate_tools + self._configure_models(check_migrate_databases=self.config.check_migrate_databases, check_migrate_tools=check_migrate_tools, config_file=config_file) + # Security helper + self._configure_security() + self._register_singleton(IdEncodingHelper, self.security) + self._register_singleton(SharedModelMapping, self.model) + self._register_singleton(GalaxyModelMapping, self.model) + self._register_singleton(scoped_session, self.model.context) + + def _shutdown_object_store(self): + self.object_store.shutdown() + + def _shutdown_model(self): + self.model.engine.dispose() + + +class UniverseApplication(StructuredApp, MinimalGalaxyApplication): + """Encapsulates the state of a Universe application""" + + def __init__(self, **kwargs) -> None: + startup_timer = ExecutionTimer() + super().__init__(fsmon=True, **kwargs) + self.haltables = [ + ("queue worker", self._shutdown_queue_worker), + ("file watcher", self._shutdown_watcher), + ("database heartbeat", self._shutdown_database_heartbeat), + ("workflow scheduler", self._shutdown_scheduling_manager), + ("object store", self._shutdown_object_store), + ("job manager", self._shutdown_job_manager), + ("application heartbeat", self._shutdown_heartbeat), + ("repository manager", self._shutdown_repo_manager), + ("database connection", self._shutdown_model), + ("application stack", self._shutdown_application_stack), + ] + self._register_singleton(StructuredApp, self) self.execution_timer_factory = self._register_singleton(ExecutionTimerFactory, ExecutionTimerFactory(self.config)) self.configure_fluent_log() # A lot of postfork initialization depends on the server name, ensure it is set immediately after forking before other postfork functions @@ -109,20 +169,8 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container self.queue_worker = self._register_singleton(GalaxyQueueWorker, GalaxyQueueWorker(self)) self._configure_tool_shed_registry() - self._configure_object_store(fsmon=True) # Setup the database engine and ORM - config_file = kwargs.get('global_conf', {}).get('__file__', None) - if config_file: - log.debug('Using "galaxy.ini" config file: %s', config_file) - check_migrate_tools = self.config.check_migrate_tools - self._configure_models(check_migrate_databases=self.config.check_migrate_databases, check_migrate_tools=check_migrate_tools, config_file=config_file) - # Security helper - self._configure_security() - self._register_singleton(IdEncodingHelper, self.security) - self._register_singleton(SharedModelMapping, self.model) - self._register_singleton(GalaxyModelMapping, self.model) - self._register_singleton(scoped_session, self.model.context) # Tag handler self.tag_handler = self._register_singleton(GalaxyTagHandler) self.user_manager = self._register_singleton(UserManager) @@ -293,67 +341,30 @@ class UniverseApplication(StructuredApp, config.ConfiguresGalaxyMixin, Container self.server_starttime = int(time.time()) # used for cachebusting log.info("Galaxy app startup finished %s" % startup_timer) - def shutdown(self): - log.debug('Shutting down') - exception = None - try: - self.queue_worker.shutdown() - except Exception as e: - exception = exception or e - log.exception("Failed to shutdown control worker cleanly") - try: - self.watchers.shutdown() - except Exception as e: - exception = exception or e - log.exception("Failed to shutdown configuration watchers cleanly") - try: - self.database_heartbeat.shutdown() - except Exception as e: - exception = exception or e - log.exception("Failed to shutdown database heartbeat cleanly") - try: - self.workflow_scheduling_manager.shutdown() - except Exception as e: - exception = exception or e - log.exception("Failed to shutdown workflow scheduling manager cleanly") - try: - self.job_manager.shutdown() - except Exception as e: - exception = exception or e - log.exception("Failed to shutdown job manager cleanly") - try: - self.object_store.shutdown() - except Exception as e: - exception = exception or e - log.exception("Failed to shutdown object store cleanly") - try: - if self.heartbeat: - self.heartbeat.shutdown() - except Exception as e: - exception = exception or e - log.exception("Failed to shutdown heartbeat cleanly") - try: - self.update_repository_manager.shutdown() - except Exception as e: - exception = exception or e - log.exception("Failed to shutdown update repository manager cleanly") + def _shutdown_queue_worker(self): + self.queue_worker.shutdown() - try: - self.model.engine.dispose() - except Exception as e: - exception = exception or e - log.exception("Failed to shutdown SA database engine cleanly") + def _shutdown_watcher(self): + self.watchers.shutdown() - try: - self.application_stack.shutdown() - except Exception as e: - exception = exception or e - log.exception("Failed to shutdown application stack interface cleanly") + def _shutdown_database_heartbeat(self): + self.database_heartbeat.shutdown() - if exception: - raise exception - else: - log.debug('Finished shutting down') + def _shutdown_scheduling_manager(self): + self.workflow_scheduling_manager.shutdown() + + def _shutdown_job_manager(self): + self.job_manager.shutdown() + + def _shutdown_heartbeat(self): + if self.heartbeat: + self.heartbeat.shutdown() + + def _shutdown_repo_manager(self): + self.update_repository_manager.shutdown() + + def _shutdown_application_stack(self): + self.application_stack.shutdown() def configure_fluent_log(self): if self.config.fluent_log: From 9cab076d1eb451dc1cf9f326362bb0555b357146 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 25 Mar 2021 18:41:41 +0100 Subject: [PATCH 19/36] Build minimal galaxy app, use in celery tasks --- lib/galaxy/app.py | 19 ++++++++++--------- lib/galaxy/celery/__init__.py | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 104b7557328..07f03614591 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -98,7 +98,7 @@ class HaltableContainer(Container): class MinimalGalaxyApplication(BasicApp, config.ConfiguresGalaxyMixin, HaltableContainer): """Encapsulates the state of a minimal Galaxy application""" - def __init__(self, fsmon=False, **kwargs) -> None: + def __init__(self, fsmon=False, configure_logging=True, **kwargs) -> None: super().__init__() self.haltables = [ ("object store", self._shutdown_object_store), @@ -117,7 +117,8 @@ class MinimalGalaxyApplication(BasicApp, config.ConfiguresGalaxyMixin, HaltableC # Read config file and check for errors self.config: Any = self._register_singleton(config.Configuration, config.Configuration(**kwargs)) self.config.check() - config.configure_logging(self.config) + if configure_logging: + config.configure_logging(self.config) self._configure_object_store(fsmon=True) config_file = kwargs.get('global_conf', {}).get('__file__', None) if config_file: @@ -131,6 +132,13 @@ class MinimalGalaxyApplication(BasicApp, config.ConfiguresGalaxyMixin, HaltableC self._register_singleton(GalaxyModelMapping, self.model) self._register_singleton(scoped_session, self.model.context) + def configure_fluent_log(self): + if self.config.fluent_log: + from galaxy.util.custom_logging.fluent_log import FluentTraceLogger + self.trace_logger = FluentTraceLogger('galaxy', self.config.fluent_host, self.config.fluent_port) + else: + self.trace_logger = None + def _shutdown_object_store(self): self.object_store.shutdown() @@ -366,13 +374,6 @@ class UniverseApplication(StructuredApp, MinimalGalaxyApplication): def _shutdown_application_stack(self): self.application_stack.shutdown() - def configure_fluent_log(self): - if self.config.fluent_log: - from galaxy.util.custom_logging.fluent_log import FluentTraceLogger - self.trace_logger = FluentTraceLogger('galaxy', self.config.fluent_host, self.config.fluent_port) - else: - self.trace_logger = None - @property def is_job_handler(self) -> bool: return (self.config.track_jobs_in_database and self.job_config.is_handler) or not self.config.track_jobs_in_database diff --git a/lib/galaxy/celery/__init__.py b/lib/galaxy/celery/__init__.py index debd75a029d..1c3cda34c46 100644 --- a/lib/galaxy/celery/__init__.py +++ b/lib/galaxy/celery/__init__.py @@ -13,7 +13,7 @@ def get_galaxy_app(): import galaxy.app if galaxy.app.app: return galaxy.app.app - galaxy_app = galaxy.app.MinimalGalaxyApplication(**get_galaxy_config()) + galaxy_app = galaxy.app.MinimalGalaxyApplication(configure_logging=False, **get_galaxy_config()) return galaxy_app From 13b61e4bba464aa2815be0bcea528d67ab91efb8 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 25 Mar 2021 18:52:22 +0100 Subject: [PATCH 20/36] Use lru_cache to speed up getting app --- lib/galaxy/celery/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/celery/__init__.py b/lib/galaxy/celery/__init__.py index 1c3cda34c46..7d02943c3cf 100644 --- a/lib/galaxy/celery/__init__.py +++ b/lib/galaxy/celery/__init__.py @@ -1,4 +1,5 @@ import os +from functools import lru_cache from celery import Celery @@ -9,6 +10,7 @@ from galaxy.util.properties import load_app_properties log = get_logger(__name__) +@lru_cache def get_galaxy_app(): import galaxy.app if galaxy.app.app: From 92b48e63d06f4eddbc88b068c6e870d92b8ee126 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 26 Mar 2021 15:36:21 +0100 Subject: [PATCH 21/36] Restructure so test collection should work --- lib/galaxy/celery/__init__.py | 37 +++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/celery/__init__.py b/lib/galaxy/celery/__init__.py index 7d02943c3cf..b02e71d5e51 100644 --- a/lib/galaxy/celery/__init__.py +++ b/lib/galaxy/celery/__init__.py @@ -10,26 +10,43 @@ from galaxy.util.properties import load_app_properties log = get_logger(__name__) -@lru_cache +@lru_cache(maxsize=1) def get_galaxy_app(): import galaxy.app if galaxy.app.app: return galaxy.app.app - galaxy_app = galaxy.app.MinimalGalaxyApplication(configure_logging=False, **get_galaxy_config()) + kwargs = get_app_properties() + kwargs['check_migrate_tools'] = False + kwargs['check_migrate_databases'] = False + galaxy_app = galaxy.app.MinimalGalaxyApplication(configure_logging=False, **kwargs) return galaxy_app -def get_galaxy_config(): - config_file = os.path.abspath(os.environ.get("GALAXY_CONFIG_FILE")) - return load_app_properties( - config_file=config_file, - config_section='galaxy', - ) +@lru_cache(maxsize=1) +def get_app_properties(): + config_file = os.environ.get("GALAXY_CONFIG_FILE") + if config_file: + return load_app_properties( + config_file=os.path.abspath(config_file), + config_section='galaxy', + ) -broker = Configuration(**get_galaxy_config()).amqp_internal_connection +@lru_cache(maxsize=1) +def get_config(): + kwargs = get_app_properties() + if kwargs: + return Configuration(**kwargs) + + +def get_broker(): + config = get_config() + if config: + return config.amqp_internal_connection + + +broker = get_broker() celery_app = Celery('galaxy', broker=broker, include=['galaxy.celery.tasks']) -log.warning(f"BROKER IS {broker}") if __name__ == '__main__': From 659651e3d86a1f8b807ce2413a2669217c8a40a3 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 26 Mar 2021 17:26:50 +0100 Subject: [PATCH 22/36] Use DI in celery task --- lib/galaxy/celery/tasks.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/celery/tasks.py b/lib/galaxy/celery/tasks.py index e8367642a1f..4985ba64a4a 100644 --- a/lib/galaxy/celery/tasks.py +++ b/lib/galaxy/celery/tasks.py @@ -1,4 +1,10 @@ +from lagom import magic_bind_to_container +from sqlalchemy.orm.scoping import ( + scoped_session, +) + from galaxy.celery import celery_app +from galaxy.model import User from galaxy.util.custom_logging import get_logger from . import get_galaxy_app @@ -11,11 +17,10 @@ def get_galaxy_context(): @celery_app.task -def recalculate_user_disk_usage(user_id=None): - app = get_galaxy_context() - sa_session = app.model.session +@magic_bind_to_container(get_galaxy_context()) +def recalculate_user_disk_usage(session: scoped_session, user_id=None): if user_id: - user = sa_session.query(app.model.User).get(user_id) + user = session.query(User).get(user_id) if user: user.calculate_and_set_disk_usage() log.info(f"New user disk usage is {user.disk_usage}") From 8bda8d41109036e832943f848d96ea1012d9e1e1 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 27 Mar 2021 11:13:38 +0100 Subject: [PATCH 23/36] Add enable_celery_tasks config option --- doc/source/admin/galaxy_options.rst | 12 ++++++++++++ lib/galaxy/config/sample/galaxy.yml.sample | 5 +++++ lib/galaxy/webapps/galaxy/config_schema.yml | 9 +++++++++ lib/galaxy/webapps/galaxy/controllers/user.py | 9 ++++++--- 4 files changed, 32 insertions(+), 3 deletions(-) diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 858fc8a21de..ddc26a4658c 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -4360,6 +4360,18 @@ :Type: str +~~~~~~~~~~~~~~~~~~~~~~~ +``enable_celery_tasks`` +~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Offload long-running tasks to a Celery task queue. Activate this + only if you have setup a Celery worker for Galaxy. For details, + see https://docs.galaxyproject.org/en/master/admin/production.html +:Default: ``false`` +:Type: bool + + ~~~~~~~~~~~~~~ ``use_pbkdf2`` ~~~~~~~~~~~~~~ diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index e630d68e858..a65ad1df8fb 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -2159,6 +2159,11 @@ galaxy: # commented out line below). #amqp_internal_connection: sqlalchemy+sqlite:///./database/control.sqlite?isolation_level=IMMEDIATE + # Offload long-running tasks to a Celery task queue. Activate this + # only if you have setup a Celery worker for Galaxy. For details, see + # https://docs.galaxyproject.org/en/master/admin/production.html + #enable_celery_tasks: false + # Allow disabling pbkdf2 hashing of passwords for legacy situations. # This should normally be left enabled unless there is a specific # reason to disable it. diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index a6fd5931f03..378e1b9bb4b 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -3183,6 +3183,15 @@ mapping: will automatically create and use a separate sqlite database located in your /database folder (indicated in the commented out line below). + enable_celery_tasks: + type: bool + default: false + required: false + desc: | + Offload long-running tasks to a Celery task queue. + Activate this only if you have setup a Celery worker for Galaxy. + For details, see https://docs.galaxyproject.org/en/master/admin/production.html + use_pbkdf2: type: bool default: true diff --git a/lib/galaxy/webapps/galaxy/controllers/user.py b/lib/galaxy/webapps/galaxy/controllers/user.py index 29bba4aa2f2..a3c042cb6f4 100644 --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -222,9 +222,12 @@ class User(BaseUIController, UsesFormDefinitionsMixin, CreatesApiKeysMixin): if message: return self.message_exception(trans, message) if trans.user: - # Queue a quota recalculation (async) task -- this takes a - # while sometimes, so we don't want to block on logout. - recalculate_user_disk_usage.delay(user_id=trans.user.id) + if trans.app.config.enable_celery_tasks: + # Queue a quota recalculation (async) task -- this takes a + # while sometimes, so we don't want to block on logout. + recalculate_user_disk_usage.delay(user_id=trans.user.id) + else: + trans.user.calculate_and_set_disk_usage() # Since logging an event requires a session, we'll log prior to ending the session trans.log_event("User logged out") trans.handle_user_logout(logout_all=logout_all) From 62cbd6ec5f7a83b736d867aff2b49ba0982f53f3 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 27 Mar 2021 11:51:46 +0100 Subject: [PATCH 24/36] Work around tool shed user controller importing celery task --- lib/galaxy/celery/__init__.py | 9 +++++---- lib/galaxy/celery/tasks.py | 8 +++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/celery/__init__.py b/lib/galaxy/celery/__init__.py index b02e71d5e51..c7a2620ae37 100644 --- a/lib/galaxy/celery/__init__.py +++ b/lib/galaxy/celery/__init__.py @@ -16,10 +16,11 @@ def get_galaxy_app(): if galaxy.app.app: return galaxy.app.app kwargs = get_app_properties() - kwargs['check_migrate_tools'] = False - kwargs['check_migrate_databases'] = False - galaxy_app = galaxy.app.MinimalGalaxyApplication(configure_logging=False, **kwargs) - return galaxy_app + if kwargs: + kwargs['check_migrate_tools'] = False + kwargs['check_migrate_databases'] = False + galaxy_app = galaxy.app.MinimalGalaxyApplication(configure_logging=False, **kwargs) + return galaxy_app @lru_cache(maxsize=1) diff --git a/lib/galaxy/celery/tasks.py b/lib/galaxy/celery/tasks.py index 4985ba64a4a..a9cf9206148 100644 --- a/lib/galaxy/celery/tasks.py +++ b/lib/galaxy/celery/tasks.py @@ -11,13 +11,15 @@ from . import get_galaxy_app log = get_logger(__name__) -def get_galaxy_context(): +def galaxy_task(func): app = get_galaxy_app() - return app + if app: + return magic_bind_to_container(app)(func) + return func @celery_app.task -@magic_bind_to_container(get_galaxy_context()) +@galaxy_task def recalculate_user_disk_usage(session: scoped_session, user_id=None): if user_id: user = session.query(User).get(user_id) From f1debeacc98cc4702f22dc0035a1d1b8cdc6c2f7 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 28 Mar 2021 12:08:04 +0200 Subject: [PATCH 25/36] Add purge dataset task --- lib/galaxy/celery/tasks.py | 8 ++++++++ lib/galaxy/managers/hdas.py | 8 +++++++- test/unit/managers/test_HDAManager.py | 2 +- test/unit/unittest_utils/galaxy_mock.py | 1 + 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/celery/tasks.py b/lib/galaxy/celery/tasks.py index a9cf9206148..01068b1a752 100644 --- a/lib/galaxy/celery/tasks.py +++ b/lib/galaxy/celery/tasks.py @@ -4,6 +4,7 @@ from sqlalchemy.orm.scoping import ( ) from galaxy.celery import celery_app +from galaxy.managers.hdas import HDAManager from galaxy.model import User from galaxy.util.custom_logging import get_logger from . import get_galaxy_app @@ -30,3 +31,10 @@ def recalculate_user_disk_usage(session: scoped_session, user_id=None): log.error("Recalculate user disk usage task failed, user %s not found" % user_id) else: log.error("Recalculate user disk usage task received without user_id.") + + +@celery_app.task +@galaxy_task +def purge_hda(hda_manager: HDAManager, hda_id): + hda = hda_manager.by_id(hda_id) + hda_manager._purge(hda) diff --git a/lib/galaxy/managers/hdas.py b/lib/galaxy/managers/hdas.py index c4d4093a452..b910a1246c5 100644 --- a/lib/galaxy/managers/hdas.py +++ b/lib/galaxy/managers/hdas.py @@ -13,6 +13,7 @@ from galaxy import ( exceptions, model ) +from galaxy.celery import celery_app from galaxy.managers import ( annotatable, datasets, @@ -135,6 +136,12 @@ class HDAManager(datasets.DatasetAssociationManager, # .... deletion and purging def purge(self, hda, flush=True): + if self.app.config.enable_celery_tasks: + celery_app.send_task('galaxy.celery.tasks.purge_hda').delay(hda_id=hda.id) + else: + self._purge(hda, flush=flush) + + def _purge(self, hda, flush=True): """ Purge this HDA and the dataset underlying it. """ @@ -146,7 +153,6 @@ class HDAManager(datasets.DatasetAssociationManager, # decrease the user's space used if quota_amount_reduction: user.adjust_total_disk_usage(-quota_amount_reduction) - return hda # .... states def error_if_uploading(self, hda): diff --git a/test/unit/managers/test_HDAManager.py b/test/unit/managers/test_HDAManager.py index 8f44aaeb47b..bdf491c1de1 100644 --- a/test/unit/managers/test_HDAManager.py +++ b/test/unit/managers/test_HDAManager.py @@ -175,7 +175,7 @@ class HDAManagerTestCase(HDATestCase): self.log("should purge an hda if config does allow") self.assertFalse(item1.purged) - self.assertEqual(self.hda_manager.purge(item1), item1) + self.hda_manager.purge(item1) self.assertTrue(item1.deleted) self.assertTrue(item1.purged) diff --git a/test/unit/unittest_utils/galaxy_mock.py b/test/unit/unittest_utils/galaxy_mock.py index 0d660173437..b060be7c340 100644 --- a/test/unit/unittest_utils/galaxy_mock.py +++ b/test/unit/unittest_utils/galaxy_mock.py @@ -139,6 +139,7 @@ class MockAppConfig(Bunch): self.security = idencoding.IdEncodingHelper(id_secret='6e46ed6483a833c100e68cc3f1d0dd76') self.database_connection = kwargs.get('database_connection', "sqlite:///:memory:") self.use_remote_user = kwargs.get('use_remote_user', False) + self.enable_celery_tasks = False self.data_dir = os.path.join(root, 'database') self.file_path = os.path.join(self.data_dir, 'files') self.jobs_directory = os.path.join(self.data_dir, 'jobs_directory') From 1afe1e12112104d44441688bf7bae18bd4a0153d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 28 Mar 2021 17:27:56 +0200 Subject: [PATCH 26/36] Break UniverseApplication into smaller parts so we can use a variant of app that loads managers in Celery. --- lib/galaxy/app.py | 83 +++++++++++++----------- lib/galaxy/celery/__init__.py | 2 +- lib/galaxy/jobs/__init__.py | 4 +- lib/galaxy/jobs/manager.py | 4 +- lib/galaxy/managers/base.py | 10 +-- lib/galaxy/managers/configuration.py | 4 +- lib/galaxy/managers/context.py | 4 +- lib/galaxy/managers/datasets.py | 6 +- lib/galaxy/managers/group_roles.py | 4 +- lib/galaxy/managers/group_users.py | 4 +- lib/galaxy/managers/groups.py | 4 +- lib/galaxy/managers/hdas.py | 10 +-- lib/galaxy/managers/hdcas.py | 10 +-- lib/galaxy/managers/histories.py | 12 ++-- lib/galaxy/managers/history_contents.py | 6 +- lib/galaxy/managers/jobs.py | 12 ++-- lib/galaxy/managers/lddas.py | 4 +- lib/galaxy/managers/metrics.py | 4 +- lib/galaxy/managers/pages.py | 10 +-- lib/galaxy/managers/remote_files.py | 4 +- lib/galaxy/managers/sharable.py | 4 +- lib/galaxy/managers/users.py | 4 +- lib/galaxy/managers/visualizations.py | 6 +- lib/galaxy/managers/workflows.py | 6 +- lib/galaxy/structured_app.py | 42 ++++++++++-- lib/galaxy/tools/cache.py | 4 +- lib/galaxy/tools/data_manager/manager.py | 4 +- lib/galaxy/visualization/genomes.py | 4 +- packages/app/requirements.txt | 1 + test/unit/unittest_utils/galaxy_mock.py | 3 +- 30 files changed, 161 insertions(+), 118 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 07f03614591..02e3d160825 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -70,7 +70,7 @@ from galaxy.web_stack import application_stack_instance, ApplicationStack from galaxy.webhooks import WebhooksRegistry from galaxy.workflow.trs_proxy import TrsProxy from .di import Container -from .structured_app import BasicApp, StructuredApp +from .structured_app import BasicApp, MininmalManagerApp, StructuredApp log = logging.getLogger(__name__) app = None @@ -146,7 +146,50 @@ class MinimalGalaxyApplication(BasicApp, config.ConfiguresGalaxyMixin, HaltableC self.model.engine.dispose() -class UniverseApplication(StructuredApp, MinimalGalaxyApplication): +class GalaxyManagerApplication(MininmalManagerApp, MinimalGalaxyApplication): + """Extends the MinimalGalaxyApplication with most managers that are not tied to a web or job handling context.""" + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._register_singleton(MininmalManagerApp, self) + self.execution_timer_factory = self._register_singleton(ExecutionTimerFactory, ExecutionTimerFactory(self.config)) + self.configure_fluent_log() + + # Tag handler + self.tag_handler = self._register_singleton(GalaxyTagHandler) + self.user_manager = self._register_singleton(UserManager) + self._register_singleton(GalaxySessionManager) + self.hda_manager = self._register_singleton(HDAManager) + self.history_manager = self._register_singleton(HistoryManager) + self.job_search = self._register_singleton(JobSearch) + self.dataset_collections_service = self._register_singleton(DatasetCollectionManager) + self.workflow_manager = self._register_singleton(WorkflowsManager) + self.workflow_contents_manager = self._register_singleton(WorkflowContentsManager) + self.library_folder_manager = self._register_singleton(FolderManager) + self.library_manager = self._register_singleton(LibraryManager) + self.role_manager = self._register_singleton(RoleManager) + + # ConfiguredFileSources + self.file_sources = self._register_singleton(ConfiguredFileSources, ConfiguredFileSources.from_app_config(self.config)) + + # We need the datatype registry for running certain tasks that modify HDAs, and to build the registry we need + # to setup the installed repositories ... this is not ideal + self._configure_tool_config_files() + self.installed_repository_manager = self._register_singleton(InstalledRepositoryManager, InstalledRepositoryManager(self)) + self._configure_datatypes_registry(self.installed_repository_manager) + self._register_singleton(Registry, self.datatypes_registry) + galaxy.model.set_datatypes_registry(self.datatypes_registry) + + self.sentry_client = None + if self.config.sentry_dsn: + + def postfork_sentry_client(): + import raven + self.sentry_client = raven.Client(self.config.sentry_dsn, transport=raven.transport.HTTPTransport) + + self.application_stack.register_postfork_function(postfork_sentry_client) + + +class UniverseApplication(StructuredApp, GalaxyManagerApplication): """Encapsulates the state of a Universe application""" def __init__(self, **kwargs) -> None: @@ -165,8 +208,6 @@ class UniverseApplication(StructuredApp, MinimalGalaxyApplication): ("application stack", self._shutdown_application_stack), ] self._register_singleton(StructuredApp, self) - self.execution_timer_factory = self._register_singleton(ExecutionTimerFactory, ExecutionTimerFactory(self.config)) - self.configure_fluent_log() # A lot of postfork initialization depends on the server name, ensure it is set immediately after forking before other postfork functions self.application_stack = self._register_singleton(ApplicationStack, application_stack_instance(app=self)) self.application_stack.register_postfork_function(self.application_stack.set_postfork_server_name, self) @@ -177,29 +218,12 @@ class UniverseApplication(StructuredApp, MinimalGalaxyApplication): self.queue_worker = self._register_singleton(GalaxyQueueWorker, GalaxyQueueWorker(self)) self._configure_tool_shed_registry() - # Setup the database engine and ORM - # Tag handler - self.tag_handler = self._register_singleton(GalaxyTagHandler) - self.user_manager = self._register_singleton(UserManager) - self._register_singleton(GalaxySessionManager) - self.hda_manager = self._register_singleton(HDAManager) - self.history_manager = self._register_singleton(HistoryManager) - self.job_search = self._register_singleton(JobSearch) - self.dataset_collections_service = self._register_singleton(DatasetCollectionManager) - self.workflow_manager = self._register_singleton(WorkflowsManager) - self.workflow_contents_manager = self._register_singleton(WorkflowContentsManager) self.dependency_resolvers_view = self._register_singleton(DependencyResolversView, DependencyResolversView(self)) self.test_data_resolver = self._register_singleton(TestDataResolver, TestDataResolver(file_dirs=self.config.tool_test_data_directories)) - self.library_folder_manager = self._register_singleton(FolderManager) - self.library_manager = self._register_singleton(LibraryManager) - self.role_manager = self._register_singleton(RoleManager) self.dynamic_tool_manager = self._register_singleton(DynamicToolManager) self.api_keys_manager = self._register_singleton(ApiKeyManager) - # ConfiguredFileSources - self.file_sources = self._register_singleton(ConfiguredFileSources, ConfiguredFileSources.from_app_config(self.config)) - # Tool Data Tables self._configure_tool_data_tables(from_shed_config=False) # Load dbkey / genome build manager @@ -225,14 +249,7 @@ class UniverseApplication(StructuredApp, MinimalGalaxyApplication): self.tool_shed_repository_cache = self._register_singleton(ToolShedRepositoryCache) # Watch various config files for immediate reload self.watchers = self._register_singleton(ConfigWatchers) - self._configure_tool_config_files() - self.installed_repository_manager = self._register_singleton(InstalledRepositoryManager, InstalledRepositoryManager(self)) - self._configure_datatypes_registry(self.installed_repository_manager) - self._register_singleton(Registry, self.datatypes_registry) - galaxy.model.set_datatypes_registry(self.datatypes_registry) - self._configure_toolbox() - # Load Data Manager self.data_managers = self._register_singleton(DataManagers) # Load the update repository manager. @@ -287,16 +304,6 @@ class UniverseApplication(StructuredApp, MinimalGalaxyApplication): self.authnz_manager = managers.AuthnzManager(self, self.config.oidc_config_file, self.config.oidc_backends_config_file) - - self.sentry_client = None - if self.config.sentry_dsn: - - def postfork_sentry_client(): - import raven - self.sentry_client = raven.Client(self.config.sentry_dsn, transport=raven.transport.HTTPTransport) - - self.application_stack.register_postfork_function(postfork_sentry_client) - # Start the job manager from galaxy.jobs import manager self.job_manager = self._register_singleton(manager.JobManager) diff --git a/lib/galaxy/celery/__init__.py b/lib/galaxy/celery/__init__.py index c7a2620ae37..da25d1e2d61 100644 --- a/lib/galaxy/celery/__init__.py +++ b/lib/galaxy/celery/__init__.py @@ -19,7 +19,7 @@ def get_galaxy_app(): if kwargs: kwargs['check_migrate_tools'] = False kwargs['check_migrate_databases'] = False - galaxy_app = galaxy.app.MinimalGalaxyApplication(configure_logging=False, **kwargs) + galaxy_app = galaxy.app.GalaxyManagerApplication(configure_logging=False, **kwargs) return galaxy_app diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index c8a6d25ed4b..185f0665c67 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -57,7 +57,7 @@ from galaxy.jobs.runners import BaseJobRunner, JobState from galaxy.metadata import get_metadata_compute_strategy from galaxy.model import store from galaxy.objectstore import ObjectStorePopulator -from galaxy.structured_app import StructuredApp +from galaxy.structured_app import MininmalManagerApp from galaxy.tool_util.deps import requirements from galaxy.tool_util.output_checker import ( check_output, @@ -301,7 +301,7 @@ class JobConfiguration(ConfiguresHandlers): """ - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): """Parse the job configuration XML. """ self.app = app diff --git a/lib/galaxy/jobs/manager.py b/lib/galaxy/jobs/manager.py index a9740288e75..7c6315f32fc 100644 --- a/lib/galaxy/jobs/manager.py +++ b/lib/galaxy/jobs/manager.py @@ -9,7 +9,7 @@ from sqlalchemy.sql.expression import null from galaxy.exceptions import HandlerAssignmentError, ToolExecutionError from galaxy.jobs import handler, NoopQueue from galaxy.model import Job -from galaxy.structured_app import StructuredApp +from galaxy.structured_app import MininmalManagerApp from galaxy.web_stack.message import JobHandlerMessage log = logging.getLogger(__name__) @@ -21,7 +21,7 @@ class JobManager: """ job_handler: handler.JobHandlerI - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): self.app = app self.job_lock = False if self.app.is_job_handler: diff --git a/lib/galaxy/managers/base.py b/lib/galaxy/managers/base.py index de18f4c279c..6d9ccfff0d4 100644 --- a/lib/galaxy/managers/base.py +++ b/lib/galaxy/managers/base.py @@ -37,7 +37,7 @@ from sqlalchemy.orm.scoping import scoped_session from galaxy import exceptions from galaxy import model from galaxy.model import tool_shed_install -from galaxy.structured_app import BasicApp, StructuredApp +from galaxy.structured_app import BasicApp, MininmalManagerApp from galaxy.util import namedtuple log = logging.getLogger(__name__) @@ -488,7 +488,7 @@ class HasAModelManager: # examples where this doesn't really work are ConfigurationSerializer (no manager) # and contents (2 managers) - def __init__(self, app: StructuredApp, manager=None, **kwargs): + def __init__(self, app: MininmalManagerApp, manager=None, **kwargs): self._manager = manager @property @@ -542,7 +542,7 @@ class ModelSerializer(HasAModelManager): default_view: Optional[str] views: Dict[str, List[str]] - def __init__(self, app: StructuredApp, **kwargs): + def __init__(self, app: MininmalManagerApp, **kwargs): """ Set up serializer map, any additional serializable keys, and views here. """ @@ -713,7 +713,7 @@ class ModelDeserializer(HasAModelManager): """ # TODO:?? a larger question is: which should be first? Deserialize then validate - or - validate then deserialize? - def __init__(self, app: StructuredApp, validator=None, **kwargs): + def __init__(self, app: MininmalManagerApp, validator=None, **kwargs): """ Set up deserializers and validator. """ @@ -904,7 +904,7 @@ class ModelFilterParser(HasAModelManager): orm_filter_parsers: Dict[str, Dict] fn_filter_parsers: Dict[str, Dict] - def __init__(self, app: StructuredApp, **kwargs): + def __init__(self, app: MininmalManagerApp, **kwargs): """ Set up serializer map, any additional serializable keys, and views here. """ diff --git a/lib/galaxy/managers/configuration.py b/lib/galaxy/managers/configuration.py index 13ddaa3b7a9..50881f0595c 100644 --- a/lib/galaxy/managers/configuration.py +++ b/lib/galaxy/managers/configuration.py @@ -15,7 +15,7 @@ from typing import ( List, ) -from galaxy.app import StructuredApp +from galaxy.app import MininmalManagerApp from galaxy.managers import base from galaxy.managers.context import ProvidesUserContext from galaxy.schema.fields import EncodedDatabaseIdField @@ -30,7 +30,7 @@ VERSION_JSON_FILE = 'version.json' class ConfigurationManager: """Interface/service object for interacting with configuration and related data.""" - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): self._app = app def get_configuration( diff --git a/lib/galaxy/managers/context.py b/lib/galaxy/managers/context.py index 4b9069a6f85..0c87db6a874 100644 --- a/lib/galaxy/managers/context.py +++ b/lib/galaxy/managers/context.py @@ -50,7 +50,7 @@ from galaxy.model import ( ) from galaxy.model.base import ModelMapping from galaxy.security.idencoding import IdEncodingHelper -from galaxy.structured_app import StructuredApp +from galaxy.structured_app import MininmalManagerApp from galaxy.util import bunch @@ -62,7 +62,7 @@ class ProvidesAppContext: """ @abc.abstractproperty - def app(self) -> StructuredApp: + def app(self) -> MininmalManagerApp: """Provide access to the Galaxy ``app`` object. """ diff --git a/lib/galaxy/managers/datasets.py b/lib/galaxy/managers/datasets.py index 0f7092e3a5e..a07e8d5f6b4 100644 --- a/lib/galaxy/managers/datasets.py +++ b/lib/galaxy/managers/datasets.py @@ -18,7 +18,7 @@ from galaxy.managers import ( secured, users ) -from galaxy.structured_app import StructuredApp +from galaxy.structured_app import MininmalManagerApp from galaxy.util.checkers import check_binary log = logging.getLogger(__name__) @@ -33,7 +33,7 @@ class DatasetManager(base.ModelManager, secured.AccessibleManagerMixin, deletabl # TODO:?? get + error_if_uploading is common pattern, should upload check be worked into access/owed? - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): super().__init__(app) self.permissions = DatasetRBACPermissions(app) # needed for admin test @@ -143,7 +143,7 @@ class DatasetRBACPermissions: class DatasetSerializer(base.ModelSerializer, deletable.PurgableSerializerMixin): model_manager_class = DatasetManager - def __init__(self, app: StructuredApp, user_manager: users.UserManager): + def __init__(self, app: MininmalManagerApp, user_manager: users.UserManager): super().__init__(app) self.dataset_manager = self.manager # needed for admin test diff --git a/lib/galaxy/managers/group_roles.py b/lib/galaxy/managers/group_roles.py index 55ad0475729..7dd9120388f 100644 --- a/lib/galaxy/managers/group_roles.py +++ b/lib/galaxy/managers/group_roles.py @@ -7,7 +7,7 @@ from typing import ( ) from galaxy import model -from galaxy.app import StructuredApp +from galaxy.app import MininmalManagerApp from galaxy.exceptions import ( ObjectNotFound, ) @@ -22,7 +22,7 @@ log = logging.getLogger(__name__) class GroupRolesManager: """Interface/service object shared by controllers for interacting with group roles.""" - def __init__(self, app: StructuredApp) -> None: + def __init__(self, app: MininmalManagerApp) -> None: self._app = app def index(self, trans: ProvidesAppContext, group_id: EncodedDatabaseIdField) -> List[Dict[str, Any]]: diff --git a/lib/galaxy/managers/group_users.py b/lib/galaxy/managers/group_users.py index 9dd194311ef..1d2b6e76792 100644 --- a/lib/galaxy/managers/group_users.py +++ b/lib/galaxy/managers/group_users.py @@ -7,7 +7,7 @@ from typing import ( ) from galaxy import model -from galaxy.app import StructuredApp +from galaxy.app import MininmalManagerApp from galaxy.exceptions import ( ObjectNotFound, ) @@ -22,7 +22,7 @@ log = logging.getLogger(__name__) class GroupUsersManager: """Interface/service object shared by controllers for interacting with group users.""" - def __init__(self, app: StructuredApp) -> None: + def __init__(self, app: MininmalManagerApp) -> None: self._app = app def index(self, trans: ProvidesAppContext, group_id: EncodedDatabaseIdField) -> List[Dict[str, Any]]: diff --git a/lib/galaxy/managers/groups.py b/lib/galaxy/managers/groups.py index 8162bf3ebfe..bc9f84d8c41 100644 --- a/lib/galaxy/managers/groups.py +++ b/lib/galaxy/managers/groups.py @@ -7,7 +7,7 @@ from typing import ( from sqlalchemy import false from galaxy import model -from galaxy.app import StructuredApp +from galaxy.app import MininmalManagerApp from galaxy.exceptions import ( Conflict, ObjectAttributeMissingException, @@ -22,7 +22,7 @@ from galaxy.web import url_for class GroupsManager: """Interface/service object shared by controllers for interacting with groups.""" - def __init__(self, app: StructuredApp) -> None: + def __init__(self, app: MininmalManagerApp) -> None: self._app = app def index(self, trans: ProvidesAppContext): diff --git a/lib/galaxy/managers/hdas.py b/lib/galaxy/managers/hdas.py index b910a1246c5..3be7434e874 100644 --- a/lib/galaxy/managers/hdas.py +++ b/lib/galaxy/managers/hdas.py @@ -21,7 +21,7 @@ from galaxy.managers import ( taggable, users, ) -from galaxy.structured_app import StructuredApp +from galaxy.structured_app import MininmalManagerApp log = logging.getLogger(__name__) @@ -42,7 +42,7 @@ class HDAManager(datasets.DatasetAssociationManager, # TODO: move what makes sense into DatasetManager # TODO: which of these are common with LDDAs and can be pushed down into DatasetAssociationManager? - def __init__(self, app: StructuredApp, user_manager: users.UserManager): + def __init__(self, app: MininmalManagerApp, user_manager: users.UserManager): """ Set up and initialize other managers needed by hdas. """ @@ -137,7 +137,7 @@ class HDAManager(datasets.DatasetAssociationManager, # .... deletion and purging def purge(self, hda, flush=True): if self.app.config.enable_celery_tasks: - celery_app.send_task('galaxy.celery.tasks.purge_hda').delay(hda_id=hda.id) + celery_app.send_task('galaxy.celery.tasks.purge_hda', kwargs={'hda_id': hda.id}) else: self._purge(hda, flush=flush) @@ -254,7 +254,7 @@ class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerialize annotatable.AnnotatableSerializerMixin): model_manager_class = HDAManager - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): super().__init__(app) self.hda_manager = self.manager @@ -513,7 +513,7 @@ class HDADeserializer(datasets.DatasetAssociationDeserializer, """ model_manager_class = HDAManager - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): super().__init__(app) self.hda_manager = self.manager diff --git a/lib/galaxy/managers/hdcas.py b/lib/galaxy/managers/hdcas.py index ac25761b6d0..805db2f9781 100644 --- a/lib/galaxy/managers/hdcas.py +++ b/lib/galaxy/managers/hdcas.py @@ -16,7 +16,7 @@ from galaxy.managers import ( taggable ) from galaxy.managers.collections_util import get_hda_and_element_identifiers -from galaxy.structured_app import StructuredApp +from galaxy.structured_app import MininmalManagerApp from galaxy.util.zipstream import ZipstreamWrapper @@ -87,7 +87,7 @@ class DCESerializer(base.ModelSerializer): Serializer for DatasetCollectionElements. """ - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): super().__init__(app) self.hda_serializer = hdas.HDASerializer(app) self.dc_serializer = DCSerializer(app, dce_serializer=self) @@ -121,7 +121,7 @@ class DCSerializer(base.ModelSerializer): Serializer for DatasetCollections. """ - def __init__(self, app: StructuredApp, dce_serializer=None): + def __init__(self, app: MininmalManagerApp, dce_serializer=None): super().__init__(app) self.dce_serializer = dce_serializer or DCESerializer(app) @@ -160,7 +160,7 @@ class DCASerializer(base.ModelSerializer): Base (abstract) Serializer class for HDCAs and LDCAs. """ - def __init__(self, app: StructuredApp, dce_serializer=None): + def __init__(self, app: MininmalManagerApp, dce_serializer=None): super().__init__(app) self.dce_serializer = dce_serializer or DCESerializer(app) @@ -215,7 +215,7 @@ class HDCASerializer( Serializer for HistoryDatasetCollectionAssociations. """ - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): super().__init__(app) self.hdca_manager = HDCAManager(app) diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index d1426d112b1..1d859efd767 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -23,7 +23,7 @@ from galaxy.managers import ( sharable ) from galaxy.schema.fields import EncodedDatabaseIdField -from galaxy.structured_app import StructuredApp +from galaxy.structured_app import MininmalManagerApp log = logging.getLogger(__name__) @@ -40,7 +40,7 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix # TODO: incorporate imp/exp (or alias to) - def __init__(self, app: StructuredApp, hda_manager: hdas.HDAManager, contents_manager: history_contents.HistoryContentsManager, contents_filters: history_contents.HistoryContentsFilters): + def __init__(self, app: MininmalManagerApp, hda_manager: hdas.HDAManager, contents_manager: history_contents.HistoryContentsManager, contents_filters: history_contents.HistoryContentsFilters): super().__init__(app) self.hda_manager = hda_manager self.contents_manager = contents_manager @@ -174,7 +174,7 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix class HistoryExportView: - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): self.app = app def get_exports(self, trans, history_id): @@ -226,7 +226,7 @@ class HistorySerializer(sharable.SharableModelSerializer, deletable.PurgableSeri model_manager_class = HistoryManager SINGLE_CHAR_ABBR = 'h' - def __init__(self, app: StructuredApp, hda_manager: hdas.HDAManager, hda_serializer: hdas.HDASerializer, history_contents_serializer: history_contents.HistoryContentsSerializer): + def __init__(self, app: MininmalManagerApp, hda_manager: hdas.HDAManager, hda_serializer: hdas.HDASerializer, history_contents_serializer: history_contents.HistoryContentsSerializer): super().__init__(app) self.history_manager = self.manager @@ -445,7 +445,7 @@ class HistoryDeserializer(sharable.SharableModelDeserializer, deletable.Purgable """ model_manager_class = HistoryManager - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): super().__init__(app) self.history_manager = self.manager @@ -482,7 +482,7 @@ class HistoriesService: and pydantic models to declare its parameters and return types. """ - def __init__(self, app: StructuredApp, manager: HistoryManager, serializer: HistorySerializer): + def __init__(self, app: MininmalManagerApp, manager: HistoryManager, serializer: HistorySerializer): self.app = app self.manager = manager self.serializer = serializer diff --git a/lib/galaxy/managers/history_contents.py b/lib/galaxy/managers/history_contents.py index 9765a7c0a95..50ee953f31e 100644 --- a/lib/galaxy/managers/history_contents.py +++ b/lib/galaxy/managers/history_contents.py @@ -32,7 +32,7 @@ from galaxy.managers import ( taggable, tools ) -from galaxy.structured_app import StructuredApp +from galaxy.structured_app import MininmalManagerApp log = logging.getLogger(__name__) @@ -73,7 +73,7 @@ class HistoryContentsManager(containers.ContainerManagerMixin): ) default_order_by = 'hid' - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): self.app = app self.contained_manager = app[self.contained_class_manager_class] self.subcontainer_manager = app[self.subcontainer_class_manager_class] @@ -427,7 +427,7 @@ class HistoryContentsSerializer(base.ModelSerializer, deletable.PurgableSerializ """ model_manager_class = HistoryContentsManager - def __init__(self, app: StructuredApp, **kwargs): + def __init__(self, app: MininmalManagerApp, **kwargs): super().__init__(app, **kwargs) self.default_view = 'summary' diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index c96de062caf..db08253505c 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -9,6 +9,7 @@ from pydantic import ( ) from sqlalchemy import and_, false, func, or_ from sqlalchemy.orm import aliased +from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.sql import select from galaxy import model @@ -21,6 +22,7 @@ from galaxy.managers.collections import DatasetCollectionManager from galaxy.managers.datasets import DatasetManager from galaxy.managers.hdas import HDAManager from galaxy.managers.lddas import LDDAManager +from galaxy.security.idencoding import IdEncodingHelper from galaxy.structured_app import StructuredApp from galaxy.util import ( defaultdict, @@ -96,17 +98,17 @@ class JobSearch: """Search for jobs using tool inputs or other jobs""" def __init__( self, - app: StructuredApp, + sa_session: scoped_session, hda_manager: HDAManager, dataset_collection_manager: DatasetCollectionManager, - ldda_manager: LDDAManager + ldda_manager: LDDAManager, + id_encoding_helper: IdEncodingHelper, ): - self.app = app - self.sa_session = app.model.context + self.sa_session = sa_session self.hda_manager = hda_manager self.dataset_collection_manager = dataset_collection_manager self.ldda_manager = ldda_manager - self.decode_id = self.app.security.decode_id + self.decode_id = id_encoding_helper.decode_id def by_tool_input(self, trans, tool_id, tool_version, param=None, param_dump=None, job_state='ok'): """Search for jobs producing same results using the 'inputs' part of a tool POST.""" diff --git a/lib/galaxy/managers/lddas.py b/lib/galaxy/managers/lddas.py index 53d2a0a9b28..fc0b18014c5 100644 --- a/lib/galaxy/managers/lddas.py +++ b/lib/galaxy/managers/lddas.py @@ -3,7 +3,7 @@ import logging from galaxy import model, util from galaxy.managers import base as manager_base from galaxy.managers.datasets import DatasetAssociationManager -from galaxy.structured_app import StructuredApp +from galaxy.structured_app import MininmalManagerApp log = logging.getLogger(__name__) @@ -14,7 +14,7 @@ class LDDAManager(DatasetAssociationManager): """ model_class = model.LibraryDatasetDatasetAssociation - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): """ Set up and initialize other managers needed by lddas. """ diff --git a/lib/galaxy/managers/metrics.py b/lib/galaxy/managers/metrics.py index 66df5c3e3cd..500acf60c1a 100644 --- a/lib/galaxy/managers/metrics.py +++ b/lib/galaxy/managers/metrics.py @@ -13,7 +13,7 @@ from pydantic import ( Field, ) -from galaxy.app import StructuredApp +from galaxy.app import MininmalManagerApp log = logging.getLogger(__name__) @@ -72,7 +72,7 @@ TimeSeriesTupleGenerator = Generator[TimeSeriesTuple, None, None] class MetricsManager: """Interface/service object shared by controllers for interacting with metrics.""" - def __init__(self, app: StructuredApp) -> None: + def __init__(self, app: MininmalManagerApp) -> None: self._app = app #: set to true to send additional debugging info to the log self.debugging = True diff --git a/lib/galaxy/managers/pages.py b/lib/galaxy/managers/pages.py index 6d7582e8a75..8b376dcebe1 100644 --- a/lib/galaxy/managers/pages.py +++ b/lib/galaxy/managers/pages.py @@ -32,7 +32,7 @@ from galaxy.managers.markdown_util import ( ) from galaxy.model.item_attrs import UsesAnnotations from galaxy.schema.fields import EncodedDatabaseIdField -from galaxy.structured_app import StructuredApp +from galaxy.structured_app import MininmalManagerApp from galaxy.util import unicodify from galaxy.util.sanitize_html import sanitize_html @@ -197,7 +197,7 @@ class PagesService: and pydantic models to declare its parameters and return types. """ - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): self.manager = PageManager(app) self.serializer = PageSerializer(app) self.shareable_service = sharable.ShareableService(self.manager, self.serializer) @@ -304,7 +304,7 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations): annotation_assoc = model.PageAnnotationAssociation rating_assoc = model.PageRatingAssociation - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): """ """ super().__init__(app) @@ -427,7 +427,7 @@ class PageSerializer(sharable.SharableModelSerializer): model_manager_class = PageManager SINGLE_CHAR_ABBR = 'p' - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): super().__init__(app) self.page_manager = PageManager(app) @@ -448,7 +448,7 @@ class PageDeserializer(sharable.SharableModelDeserializer): """ model_manager_class = PageManager - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): super().__init__(app) self.page_manager = self.manager diff --git a/lib/galaxy/managers/remote_files.py b/lib/galaxy/managers/remote_files.py index a7dc50a550e..dabd2d0255d 100644 --- a/lib/galaxy/managers/remote_files.py +++ b/lib/galaxy/managers/remote_files.py @@ -12,7 +12,7 @@ from typing import ( from pydantic.tools import parse_obj_as from galaxy import exceptions -from galaxy.app import StructuredApp +from galaxy.app import MininmalManagerApp from galaxy.files import ( ConfiguredFileSources, ProvidesUserFileSourcesUserContext, @@ -37,7 +37,7 @@ class RemoteFilesManager: Interface/service object for interacting with remote files. """ - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): self._app = app def index( diff --git a/lib/galaxy/managers/sharable.py b/lib/galaxy/managers/sharable.py index 1ec13776546..ad8c15f9791 100644 --- a/lib/galaxy/managers/sharable.py +++ b/lib/galaxy/managers/sharable.py @@ -34,7 +34,7 @@ from galaxy.managers import ( ) from galaxy.model import UserShareAssociation from galaxy.schema.fields import EncodedDatabaseIdField -from galaxy.structured_app import StructuredApp +from galaxy.structured_app import MininmalManagerApp from galaxy.util import ready_name_for_url log = logging.getLogger(__name__) @@ -51,7 +51,7 @@ class SharableModelManager(base.ModelManager, secured.OwnableManagerMixin, secur #: the single character abbreviation used in username_and_slug: e.g. 'h' for histories: u/user/h/slug SINGLE_CHAR_ABBR: Optional[str] = None - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): super().__init__(app) # user manager is needed to check access/ownership/admin self.user_manager = users.UserManager(app) diff --git a/lib/galaxy/managers/users.py b/lib/galaxy/managers/users.py index e8b0493bc31..42cd31f5541 100644 --- a/lib/galaxy/managers/users.py +++ b/lib/galaxy/managers/users.py @@ -30,7 +30,7 @@ from galaxy.security.validate_user_input import ( validate_password, validate_publicname ) -from galaxy.structured_app import BasicApp, StructuredApp +from galaxy.structured_app import BasicApp, MininmalManagerApp from galaxy.util.hash_util import new_secure_hash from galaxy.web import url_for @@ -611,7 +611,7 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): class UserSerializer(base.ModelSerializer, deletable.PurgableSerializerMixin): model_manager_class = UserManager - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): """ Convert a User and associated data to a dictionary representation. """ diff --git a/lib/galaxy/managers/visualizations.py b/lib/galaxy/managers/visualizations.py index 5c3c55be16d..b49e2f11e1c 100644 --- a/lib/galaxy/managers/visualizations.py +++ b/lib/galaxy/managers/visualizations.py @@ -10,7 +10,7 @@ from typing import Optional from galaxy import model from galaxy.managers import sharable from galaxy.schema.fields import EncodedDatabaseIdField -from galaxy.structured_app import StructuredApp +from galaxy.structured_app import MininmalManagerApp log = logging.getLogger(__name__) @@ -43,7 +43,7 @@ class VisualizationSerializer(sharable.SharableModelSerializer): model_manager_class = VisualizationManager SINGLE_CHAR_ABBR = 'v' - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): super().__init__(app) self.visualization_manager = self.manager @@ -82,7 +82,7 @@ class VisualizationsService: and pydantic models to declare its parameters and return types. """ - def __init__(self, app: StructuredApp, manager: VisualizationManager, serializer: VisualizationSerializer): + def __init__(self, app: MininmalManagerApp, manager: VisualizationManager, serializer: VisualizationSerializer): self.app = app self.manager = manager self.serializer = serializer diff --git a/lib/galaxy/managers/workflows.py b/lib/galaxy/managers/workflows.py index 0877596279d..95a280c3e57 100644 --- a/lib/galaxy/managers/workflows.py +++ b/lib/galaxy/managers/workflows.py @@ -26,7 +26,7 @@ from galaxy import ( ) from galaxy.jobs.actions.post import ActionBox from galaxy.model.item_attrs import UsesAnnotations -from galaxy.structured_app import StructuredApp +from galaxy.structured_app import MininmalManagerApp from galaxy.tools.parameters import ( params_to_incoming, visit_input_values @@ -69,7 +69,7 @@ class WorkflowsManager: the galaxy.workflow module. """ - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): self.app = app def get_stored_workflow(self, trans, workflow_id, by_stored_id=True): @@ -306,7 +306,7 @@ CreatedWorkflow = namedtuple("CreatedWorkflow", ["stored_workflow", "workflow", class WorkflowContentsManager(UsesAnnotations): - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): self.app = app self._resource_mapper_function = get_resource_mapper_function(app) diff --git a/lib/galaxy/structured_app.py b/lib/galaxy/structured_app.py index 7728929a0ee..df3d67b7571 100644 --- a/lib/galaxy/structured_app.py +++ b/lib/galaxy/structured_app.py @@ -41,7 +41,43 @@ class BasicApp(Container): datatypes_registry: Registry -class StructuredApp(BasicApp): +class MinimalApp(BasicApp): + is_webapp: bool # is_webapp will be set to true when building WSGI app + new_installation: bool + tag_handler: GalaxyTagHandler + model: GalaxyModelMapping + install_model: ModelMapping + security_agent: GalaxyRBACAgent + host_security_agent: HostAgent + + +class MininmalManagerApp(MinimalApp): + is_webapp: bool # is_webapp will be set to true when building WSGI app + new_installation: bool + tag_handler: GalaxyTagHandler + file_sources: ConfiguredFileSources + genome_builds: GenomeBuilds + model: GalaxyModelMapping + install_model: ModelMapping + security_agent: GalaxyRBACAgent + host_security_agent: HostAgent + dataset_collections_service: Any # 'galaxy.managers.collections.DatasetCollectionManager' + history_manager: Any # 'galaxy.managers.histories.HistoryManager' + hda_manager: Any # 'galaxy.managers.hdas.HDAManager' + workflow_manager: Any # 'galaxy.managers.workflows.WorkflowsManager' + workflow_contents_manager: Any # 'galaxy.managers.workflows.WorkflowContentsManager' + library_folder_manager: Any # 'galaxy.managers.folders.FolderManager' + library_manager: Any # 'galaxy.managers.libraries.LibraryManager' + role_manager: Any # 'galaxy.managers.roles.RoleManager' + installed_repository_manager: Any # 'galaxy.tool_shed.galaxy_install.installed_repository_manager.InstalledRepositoryManager' + user_manager: Any + + @property + def is_job_handler(self) -> bool: + pass + + +class StructuredApp(MininmalManagerApp): """Interface defining typed description of the Galaxy UniverseApplication. Ideally nothing that depends on StructuredApp should require @@ -92,7 +128,3 @@ class StructuredApp(BasicApp): job_manager: Any # galaxy.jobs.manager.JobManager user_manager: Any api_keys_manager: Any - - @property - def is_job_handler(self) -> bool: - pass diff --git a/lib/galaxy/tools/cache.py b/lib/galaxy/tools/cache.py index 94e0b6d2996..b6eff3f9e10 100644 --- a/lib/galaxy/tools/cache.py +++ b/lib/galaxy/tools/cache.py @@ -16,7 +16,7 @@ from sqlalchemy.orm import ( from sqlitedict import SqliteDict from galaxy.model.tool_shed_install import ToolShedRepository -from galaxy.structured_app import StructuredApp +from galaxy.structured_app import MininmalManagerApp from galaxy.tools.toolbox.base import ToolConfRepository from galaxy.util import unicodify from galaxy.util.hash_util import md5_hash_file @@ -286,7 +286,7 @@ class ToolShedRepositoryCache: repositories: List[ToolShedRepository] repos_by_tuple: Dict[Tuple[str, str, str], List[ToolConfRepository]] - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): self.app = app # Contains ToolConfRepository objects created from shed_tool_conf.xml entries self.local_repositories = [] diff --git a/lib/galaxy/tools/data_manager/manager.py b/lib/galaxy/tools/data_manager/manager.py index 4805f1320ab..849a7fd4529 100644 --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -5,7 +5,7 @@ import os from typing import Dict from galaxy import util -from galaxy.structured_app import StructuredApp +from galaxy.structured_app import MininmalManagerApp from galaxy.tools.data import TabularToolDataTable from galaxy.util.template import fill_template @@ -20,7 +20,7 @@ class DataManagers: data_managers: Dict[str, 'DataManager'] managed_data_tables: Dict[str, 'DataManager'] - def __init__(self, app: StructuredApp, xml_filename=None): + def __init__(self, app: MininmalManagerApp, xml_filename=None): self.app = app self.data_managers = {} self.managed_data_tables = {} diff --git a/lib/galaxy/visualization/genomes.py b/lib/galaxy/visualization/genomes.py index 1737b9e1802..8738432c75a 100644 --- a/lib/galaxy/visualization/genomes.py +++ b/lib/galaxy/visualization/genomes.py @@ -11,7 +11,7 @@ from galaxy.exceptions import ( ObjectNotFound, ReferenceDataError, ) -from galaxy.structured_app import StructuredApp +from galaxy.structured_app import MininmalManagerApp from galaxy.util.bunch import Bunch log = logging.getLogger(__name__) @@ -197,7 +197,7 @@ class Genomes: Provides information about available genome data and methods for manipulating that data. """ - def __init__(self, app: StructuredApp): + def __init__(self, app: MininmalManagerApp): self.app = app # Create list of genomes from app.genome_builds self.genomes: Dict[str, Genome] = {} diff --git a/packages/app/requirements.txt b/packages/app/requirements.txt index 4be8956dc10..c7ff5fbe1b6 100644 --- a/packages/app/requirements.txt +++ b/packages/app/requirements.txt @@ -5,6 +5,7 @@ galaxy-tool-util galaxy-web-framework galaxy-web-stack +celery kombu Beaker pykwalify diff --git a/test/unit/unittest_utils/galaxy_mock.py b/test/unit/unittest_utils/galaxy_mock.py index b060be7c340..a3779bded3a 100644 --- a/test/unit/unittest_utils/galaxy_mock.py +++ b/test/unit/unittest_utils/galaxy_mock.py @@ -21,7 +21,7 @@ from galaxy.model import mapping, tags from galaxy.model.base import SharedModelMapping from galaxy.model.mapping import GalaxyModelMapping from galaxy.security import idencoding -from galaxy.structured_app import BasicApp, StructuredApp +from galaxy.structured_app import BasicApp, MininmalManagerApp, StructuredApp from galaxy.tool_util.deps.containers import NullContainerFinder from galaxy.util import StructuredExecutionTimer from galaxy.util.bunch import Bunch @@ -67,6 +67,7 @@ class MockApp(di.Container): def __init__(self, config=None, **kwargs): super().__init__() self[BasicApp] = self + self[MininmalManagerApp] = self self[StructuredApp] = self self.config = config or MockAppConfig(**kwargs) self.security = self.config.security From cc586094888e5dae1ecd39d61271baeeb906414b Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 28 Mar 2021 18:15:03 +0200 Subject: [PATCH 27/36] Let jop handler queue stop job Unless jobs are not tracked in the database (a contender for deprecation ?). --- lib/galaxy/managers/datasets.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/managers/datasets.py b/lib/galaxy/managers/datasets.py index a07e8d5f6b4..1dcbd0b3ab5 100644 --- a/lib/galaxy/managers/datasets.py +++ b/lib/galaxy/managers/datasets.py @@ -302,8 +302,10 @@ class DatasetAssociationManager(base.ModelManager, if not job.finished: # Are *all* of the job's other output datasets deleted? if job.check_if_output_datasets_deleted(): - job.mark_deleted(self.app.config.track_jobs_in_database) - self.app.job_manager.stop(job) + track_jobs_in_database = self.app.config.track_jobs_in_database + job.mark_deleted(track_jobs_in_database) + if not track_jobs_in_database: + self.app.job_manager.stop(job) return True return False From 855759cb75ce7f72650671dda3cd8bb8cf73c12a Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 28 Mar 2021 18:15:56 +0200 Subject: [PATCH 28/36] Ensure connection pool is closed when forking --- lib/galaxy/app.py | 2 -- lib/galaxy/model/orm/engine_factory.py | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 02e3d160825..e2b46bd104c 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -347,8 +347,6 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): # Delay toolbox index until after startup self.application_stack.register_postfork_function(lambda: send_local_control_task(self, 'rebuild_toolbox_search_index')) - self.model.engine.dispose() - # Inject url_for for components to more easily optionally depend # on url_for. self.url_for = url_for diff --git a/lib/galaxy/model/orm/engine_factory.py b/lib/galaxy/model/orm/engine_factory.py index 8d23f4288f7..5748525b421 100644 --- a/lib/galaxy/model/orm/engine_factory.py +++ b/lib/galaxy/model/orm/engine_factory.py @@ -3,6 +3,7 @@ import logging import os import threading import time +from multiprocessing.util import register_after_fork from sqlalchemy import ( create_engine, @@ -87,4 +88,5 @@ def build_engine(url, engine_options, database_query_profiling_proxy=False, trac # Create the database engine engine = create_engine(url, **engine_options) + register_after_fork(engine, lambda e: e.dispose()) return engine From 3b750431726ae4a2eb6b45dcf37878a4140cc23f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 29 Mar 2021 10:26:53 +0200 Subject: [PATCH 29/36] Don't override tempfile.tempdir when importing celery app Fixes ``` Traceback (most recent call last): File "./scripts/functional_tests.py", line 127, in driver_util.drive_test(find_test_driver()) File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy_test/driver/driver_util.py", line 1151, in drive_test sys.exit(test_driver.run()) File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy_test/driver/driver_util.py", line 940, in run self.setup() File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy_test/driver/driver_util.py", line 1007, in setup self._configure(config_object) File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy_test/driver/driver_util.py", line 985, in _configure self.galaxy_test_tmp_dir = get_galaxy_test_tmp_dir() File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy_test/driver/driver_util.py", line 106, in get_galaxy_test_tmp_dir galaxy_test_tmp_dir = tempfile.mkdtemp() File "/opt/hostedtoolcache/Python/3.7.10/x64/lib/python3.7/tempfile.py", line 366, in mkdtemp _os.mkdir(file, 0o700) FileNotFoundError: [Errno 2] No such file or directory: '/tmp/tmpi834284l/tmp/tmpts6ajx8g' ``` --- lib/galaxy/celery/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/celery/__init__.py b/lib/galaxy/celery/__init__.py index da25d1e2d61..59d4bc410c7 100644 --- a/lib/galaxy/celery/__init__.py +++ b/lib/galaxy/celery/__init__.py @@ -37,6 +37,7 @@ def get_app_properties(): def get_config(): kwargs = get_app_properties() if kwargs: + kwargs['override_tempdir'] = False return Configuration(**kwargs) From 908708c3c00c5b58bf2208be7391ea14b70fe7c9 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 29 Mar 2021 12:48:16 +0200 Subject: [PATCH 30/36] Add celery to galaxy-app --- packages/app/galaxy/celery | 1 + 1 file changed, 1 insertion(+) create mode 120000 packages/app/galaxy/celery diff --git a/packages/app/galaxy/celery b/packages/app/galaxy/celery new file mode 120000 index 00000000000..6c7157a6e38 --- /dev/null +++ b/packages/app/galaxy/celery @@ -0,0 +1 @@ +../../../lib/galaxy/celery \ No newline at end of file From 72b84c9ec8806f5cbc23bb651a8f4e655577655e Mon Sep 17 00:00:00 2001 From: Dannon Date: Mon, 29 Mar 2021 09:00:08 -0400 Subject: [PATCH 31/36] Update lib/galaxy/structured_app.py Co-authored-by: Nicola Soranzo --- lib/galaxy/structured_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/structured_app.py b/lib/galaxy/structured_app.py index df3d67b7571..d179d98ec9f 100644 --- a/lib/galaxy/structured_app.py +++ b/lib/galaxy/structured_app.py @@ -51,7 +51,7 @@ class MinimalApp(BasicApp): host_security_agent: HostAgent -class MininmalManagerApp(MinimalApp): +class MinimalManagerApp(MinimalApp): is_webapp: bool # is_webapp will be set to true when building WSGI app new_installation: bool tag_handler: GalaxyTagHandler From aa43f4fd37f7eac987d9de4997649aea5b343df2 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Mon, 29 Mar 2021 09:07:17 -0400 Subject: [PATCH 32/36] Use celery tasks directly --- lib/galaxy/managers/hdas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/managers/hdas.py b/lib/galaxy/managers/hdas.py index 3be7434e874..cd70aa6c9fd 100644 --- a/lib/galaxy/managers/hdas.py +++ b/lib/galaxy/managers/hdas.py @@ -13,7 +13,7 @@ from galaxy import ( exceptions, model ) -from galaxy.celery import celery_app +from galaxy.celery.tasks import purge_hda from galaxy.managers import ( annotatable, datasets, @@ -137,7 +137,7 @@ class HDAManager(datasets.DatasetAssociationManager, # .... deletion and purging def purge(self, hda, flush=True): if self.app.config.enable_celery_tasks: - celery_app.send_task('galaxy.celery.tasks.purge_hda', kwargs={'hda_id': hda.id}) + purge_hda.delay(hda_id=hda.id) else: self._purge(hda, flush=flush) From 857171e7874c98a587335dea937e03051bbcb702 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Mon, 29 Mar 2021 09:09:57 -0400 Subject: [PATCH 33/36] Globally swap MininmalManagerApp to MinimalManagerApp (thanks for noticing the first one @nsoranzo -- I am surprised how many of these were in here) --- lib/galaxy/app.py | 6 +++--- lib/galaxy/jobs/__init__.py | 4 ++-- lib/galaxy/jobs/manager.py | 4 ++-- lib/galaxy/managers/base.py | 10 +++++----- lib/galaxy/managers/configuration.py | 4 ++-- lib/galaxy/managers/context.py | 4 ++-- lib/galaxy/managers/datasets.py | 6 +++--- lib/galaxy/managers/group_roles.py | 4 ++-- lib/galaxy/managers/group_users.py | 4 ++-- lib/galaxy/managers/groups.py | 4 ++-- lib/galaxy/managers/hdas.py | 8 ++++---- lib/galaxy/managers/hdcas.py | 10 +++++----- lib/galaxy/managers/histories.py | 12 ++++++------ lib/galaxy/managers/history_contents.py | 6 +++--- lib/galaxy/managers/lddas.py | 4 ++-- lib/galaxy/managers/metrics.py | 4 ++-- lib/galaxy/managers/pages.py | 10 +++++----- lib/galaxy/managers/remote_files.py | 4 ++-- lib/galaxy/managers/sharable.py | 4 ++-- lib/galaxy/managers/users.py | 4 ++-- lib/galaxy/managers/visualizations.py | 6 +++--- lib/galaxy/managers/workflows.py | 6 +++--- lib/galaxy/structured_app.py | 2 +- lib/galaxy/tools/cache.py | 4 ++-- lib/galaxy/tools/data_manager/manager.py | 4 ++-- lib/galaxy/visualization/genomes.py | 4 ++-- test/unit/unittest_utils/galaxy_mock.py | 4 ++-- 27 files changed, 73 insertions(+), 73 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index e2b46bd104c..1549a7a9e82 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -70,7 +70,7 @@ from galaxy.web_stack import application_stack_instance, ApplicationStack from galaxy.webhooks import WebhooksRegistry from galaxy.workflow.trs_proxy import TrsProxy from .di import Container -from .structured_app import BasicApp, MininmalManagerApp, StructuredApp +from .structured_app import BasicApp, MinimalManagerApp, StructuredApp log = logging.getLogger(__name__) app = None @@ -146,11 +146,11 @@ class MinimalGalaxyApplication(BasicApp, config.ConfiguresGalaxyMixin, HaltableC self.model.engine.dispose() -class GalaxyManagerApplication(MininmalManagerApp, MinimalGalaxyApplication): +class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication): """Extends the MinimalGalaxyApplication with most managers that are not tied to a web or job handling context.""" def __init__(self, **kwargs): super().__init__(**kwargs) - self._register_singleton(MininmalManagerApp, self) + self._register_singleton(MinimalManagerApp, self) self.execution_timer_factory = self._register_singleton(ExecutionTimerFactory, ExecutionTimerFactory(self.config)) self.configure_fluent_log() diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 185f0665c67..2ef3f7a3f63 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -57,7 +57,7 @@ from galaxy.jobs.runners import BaseJobRunner, JobState from galaxy.metadata import get_metadata_compute_strategy from galaxy.model import store from galaxy.objectstore import ObjectStorePopulator -from galaxy.structured_app import MininmalManagerApp +from galaxy.structured_app import MinimalManagerApp from galaxy.tool_util.deps import requirements from galaxy.tool_util.output_checker import ( check_output, @@ -301,7 +301,7 @@ class JobConfiguration(ConfiguresHandlers): """ - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): """Parse the job configuration XML. """ self.app = app diff --git a/lib/galaxy/jobs/manager.py b/lib/galaxy/jobs/manager.py index 7c6315f32fc..fab77182b5d 100644 --- a/lib/galaxy/jobs/manager.py +++ b/lib/galaxy/jobs/manager.py @@ -9,7 +9,7 @@ from sqlalchemy.sql.expression import null from galaxy.exceptions import HandlerAssignmentError, ToolExecutionError from galaxy.jobs import handler, NoopQueue from galaxy.model import Job -from galaxy.structured_app import MininmalManagerApp +from galaxy.structured_app import MinimalManagerApp from galaxy.web_stack.message import JobHandlerMessage log = logging.getLogger(__name__) @@ -21,7 +21,7 @@ class JobManager: """ job_handler: handler.JobHandlerI - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): self.app = app self.job_lock = False if self.app.is_job_handler: diff --git a/lib/galaxy/managers/base.py b/lib/galaxy/managers/base.py index 6d9ccfff0d4..bac9e9198ab 100644 --- a/lib/galaxy/managers/base.py +++ b/lib/galaxy/managers/base.py @@ -37,7 +37,7 @@ from sqlalchemy.orm.scoping import scoped_session from galaxy import exceptions from galaxy import model from galaxy.model import tool_shed_install -from galaxy.structured_app import BasicApp, MininmalManagerApp +from galaxy.structured_app import BasicApp, MinimalManagerApp from galaxy.util import namedtuple log = logging.getLogger(__name__) @@ -488,7 +488,7 @@ class HasAModelManager: # examples where this doesn't really work are ConfigurationSerializer (no manager) # and contents (2 managers) - def __init__(self, app: MininmalManagerApp, manager=None, **kwargs): + def __init__(self, app: MinimalManagerApp, manager=None, **kwargs): self._manager = manager @property @@ -542,7 +542,7 @@ class ModelSerializer(HasAModelManager): default_view: Optional[str] views: Dict[str, List[str]] - def __init__(self, app: MininmalManagerApp, **kwargs): + def __init__(self, app: MinimalManagerApp, **kwargs): """ Set up serializer map, any additional serializable keys, and views here. """ @@ -713,7 +713,7 @@ class ModelDeserializer(HasAModelManager): """ # TODO:?? a larger question is: which should be first? Deserialize then validate - or - validate then deserialize? - def __init__(self, app: MininmalManagerApp, validator=None, **kwargs): + def __init__(self, app: MinimalManagerApp, validator=None, **kwargs): """ Set up deserializers and validator. """ @@ -904,7 +904,7 @@ class ModelFilterParser(HasAModelManager): orm_filter_parsers: Dict[str, Dict] fn_filter_parsers: Dict[str, Dict] - def __init__(self, app: MininmalManagerApp, **kwargs): + def __init__(self, app: MinimalManagerApp, **kwargs): """ Set up serializer map, any additional serializable keys, and views here. """ diff --git a/lib/galaxy/managers/configuration.py b/lib/galaxy/managers/configuration.py index 50881f0595c..cbf2bd1be09 100644 --- a/lib/galaxy/managers/configuration.py +++ b/lib/galaxy/managers/configuration.py @@ -15,7 +15,7 @@ from typing import ( List, ) -from galaxy.app import MininmalManagerApp +from galaxy.app import MinimalManagerApp from galaxy.managers import base from galaxy.managers.context import ProvidesUserContext from galaxy.schema.fields import EncodedDatabaseIdField @@ -30,7 +30,7 @@ VERSION_JSON_FILE = 'version.json' class ConfigurationManager: """Interface/service object for interacting with configuration and related data.""" - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): self._app = app def get_configuration( diff --git a/lib/galaxy/managers/context.py b/lib/galaxy/managers/context.py index 0c87db6a874..cbdf15cbfb5 100644 --- a/lib/galaxy/managers/context.py +++ b/lib/galaxy/managers/context.py @@ -50,7 +50,7 @@ from galaxy.model import ( ) from galaxy.model.base import ModelMapping from galaxy.security.idencoding import IdEncodingHelper -from galaxy.structured_app import MininmalManagerApp +from galaxy.structured_app import MinimalManagerApp from galaxy.util import bunch @@ -62,7 +62,7 @@ class ProvidesAppContext: """ @abc.abstractproperty - def app(self) -> MininmalManagerApp: + def app(self) -> MinimalManagerApp: """Provide access to the Galaxy ``app`` object. """ diff --git a/lib/galaxy/managers/datasets.py b/lib/galaxy/managers/datasets.py index 1dcbd0b3ab5..483e13004fc 100644 --- a/lib/galaxy/managers/datasets.py +++ b/lib/galaxy/managers/datasets.py @@ -18,7 +18,7 @@ from galaxy.managers import ( secured, users ) -from galaxy.structured_app import MininmalManagerApp +from galaxy.structured_app import MinimalManagerApp from galaxy.util.checkers import check_binary log = logging.getLogger(__name__) @@ -33,7 +33,7 @@ class DatasetManager(base.ModelManager, secured.AccessibleManagerMixin, deletabl # TODO:?? get + error_if_uploading is common pattern, should upload check be worked into access/owed? - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): super().__init__(app) self.permissions = DatasetRBACPermissions(app) # needed for admin test @@ -143,7 +143,7 @@ class DatasetRBACPermissions: class DatasetSerializer(base.ModelSerializer, deletable.PurgableSerializerMixin): model_manager_class = DatasetManager - def __init__(self, app: MininmalManagerApp, user_manager: users.UserManager): + def __init__(self, app: MinimalManagerApp, user_manager: users.UserManager): super().__init__(app) self.dataset_manager = self.manager # needed for admin test diff --git a/lib/galaxy/managers/group_roles.py b/lib/galaxy/managers/group_roles.py index 7dd9120388f..a6e488272d0 100644 --- a/lib/galaxy/managers/group_roles.py +++ b/lib/galaxy/managers/group_roles.py @@ -7,7 +7,7 @@ from typing import ( ) from galaxy import model -from galaxy.app import MininmalManagerApp +from galaxy.app import MinimalManagerApp from galaxy.exceptions import ( ObjectNotFound, ) @@ -22,7 +22,7 @@ log = logging.getLogger(__name__) class GroupRolesManager: """Interface/service object shared by controllers for interacting with group roles.""" - def __init__(self, app: MininmalManagerApp) -> None: + def __init__(self, app: MinimalManagerApp) -> None: self._app = app def index(self, trans: ProvidesAppContext, group_id: EncodedDatabaseIdField) -> List[Dict[str, Any]]: diff --git a/lib/galaxy/managers/group_users.py b/lib/galaxy/managers/group_users.py index 1d2b6e76792..1bbef7ec4a2 100644 --- a/lib/galaxy/managers/group_users.py +++ b/lib/galaxy/managers/group_users.py @@ -7,7 +7,7 @@ from typing import ( ) from galaxy import model -from galaxy.app import MininmalManagerApp +from galaxy.app import MinimalManagerApp from galaxy.exceptions import ( ObjectNotFound, ) @@ -22,7 +22,7 @@ log = logging.getLogger(__name__) class GroupUsersManager: """Interface/service object shared by controllers for interacting with group users.""" - def __init__(self, app: MininmalManagerApp) -> None: + def __init__(self, app: MinimalManagerApp) -> None: self._app = app def index(self, trans: ProvidesAppContext, group_id: EncodedDatabaseIdField) -> List[Dict[str, Any]]: diff --git a/lib/galaxy/managers/groups.py b/lib/galaxy/managers/groups.py index bc9f84d8c41..a35abb4838c 100644 --- a/lib/galaxy/managers/groups.py +++ b/lib/galaxy/managers/groups.py @@ -7,7 +7,7 @@ from typing import ( from sqlalchemy import false from galaxy import model -from galaxy.app import MininmalManagerApp +from galaxy.app import MinimalManagerApp from galaxy.exceptions import ( Conflict, ObjectAttributeMissingException, @@ -22,7 +22,7 @@ from galaxy.web import url_for class GroupsManager: """Interface/service object shared by controllers for interacting with groups.""" - def __init__(self, app: MininmalManagerApp) -> None: + def __init__(self, app: MinimalManagerApp) -> None: self._app = app def index(self, trans: ProvidesAppContext): diff --git a/lib/galaxy/managers/hdas.py b/lib/galaxy/managers/hdas.py index cd70aa6c9fd..d87d9f55f1e 100644 --- a/lib/galaxy/managers/hdas.py +++ b/lib/galaxy/managers/hdas.py @@ -21,7 +21,7 @@ from galaxy.managers import ( taggable, users, ) -from galaxy.structured_app import MininmalManagerApp +from galaxy.structured_app import MinimalManagerApp log = logging.getLogger(__name__) @@ -42,7 +42,7 @@ class HDAManager(datasets.DatasetAssociationManager, # TODO: move what makes sense into DatasetManager # TODO: which of these are common with LDDAs and can be pushed down into DatasetAssociationManager? - def __init__(self, app: MininmalManagerApp, user_manager: users.UserManager): + def __init__(self, app: MinimalManagerApp, user_manager: users.UserManager): """ Set up and initialize other managers needed by hdas. """ @@ -254,7 +254,7 @@ class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerialize annotatable.AnnotatableSerializerMixin): model_manager_class = HDAManager - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): super().__init__(app) self.hda_manager = self.manager @@ -513,7 +513,7 @@ class HDADeserializer(datasets.DatasetAssociationDeserializer, """ model_manager_class = HDAManager - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): super().__init__(app) self.hda_manager = self.manager diff --git a/lib/galaxy/managers/hdcas.py b/lib/galaxy/managers/hdcas.py index 805db2f9781..d810fcc9757 100644 --- a/lib/galaxy/managers/hdcas.py +++ b/lib/galaxy/managers/hdcas.py @@ -16,7 +16,7 @@ from galaxy.managers import ( taggable ) from galaxy.managers.collections_util import get_hda_and_element_identifiers -from galaxy.structured_app import MininmalManagerApp +from galaxy.structured_app import MinimalManagerApp from galaxy.util.zipstream import ZipstreamWrapper @@ -87,7 +87,7 @@ class DCESerializer(base.ModelSerializer): Serializer for DatasetCollectionElements. """ - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): super().__init__(app) self.hda_serializer = hdas.HDASerializer(app) self.dc_serializer = DCSerializer(app, dce_serializer=self) @@ -121,7 +121,7 @@ class DCSerializer(base.ModelSerializer): Serializer for DatasetCollections. """ - def __init__(self, app: MininmalManagerApp, dce_serializer=None): + def __init__(self, app: MinimalManagerApp, dce_serializer=None): super().__init__(app) self.dce_serializer = dce_serializer or DCESerializer(app) @@ -160,7 +160,7 @@ class DCASerializer(base.ModelSerializer): Base (abstract) Serializer class for HDCAs and LDCAs. """ - def __init__(self, app: MininmalManagerApp, dce_serializer=None): + def __init__(self, app: MinimalManagerApp, dce_serializer=None): super().__init__(app) self.dce_serializer = dce_serializer or DCESerializer(app) @@ -215,7 +215,7 @@ class HDCASerializer( Serializer for HistoryDatasetCollectionAssociations. """ - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): super().__init__(app) self.hdca_manager = HDCAManager(app) diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index 1d859efd767..b3975772f94 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -23,7 +23,7 @@ from galaxy.managers import ( sharable ) from galaxy.schema.fields import EncodedDatabaseIdField -from galaxy.structured_app import MininmalManagerApp +from galaxy.structured_app import MinimalManagerApp log = logging.getLogger(__name__) @@ -40,7 +40,7 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix # TODO: incorporate imp/exp (or alias to) - def __init__(self, app: MininmalManagerApp, hda_manager: hdas.HDAManager, contents_manager: history_contents.HistoryContentsManager, contents_filters: history_contents.HistoryContentsFilters): + def __init__(self, app: MinimalManagerApp, hda_manager: hdas.HDAManager, contents_manager: history_contents.HistoryContentsManager, contents_filters: history_contents.HistoryContentsFilters): super().__init__(app) self.hda_manager = hda_manager self.contents_manager = contents_manager @@ -174,7 +174,7 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix class HistoryExportView: - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): self.app = app def get_exports(self, trans, history_id): @@ -226,7 +226,7 @@ class HistorySerializer(sharable.SharableModelSerializer, deletable.PurgableSeri model_manager_class = HistoryManager SINGLE_CHAR_ABBR = 'h' - def __init__(self, app: MininmalManagerApp, hda_manager: hdas.HDAManager, hda_serializer: hdas.HDASerializer, history_contents_serializer: history_contents.HistoryContentsSerializer): + def __init__(self, app: MinimalManagerApp, hda_manager: hdas.HDAManager, hda_serializer: hdas.HDASerializer, history_contents_serializer: history_contents.HistoryContentsSerializer): super().__init__(app) self.history_manager = self.manager @@ -445,7 +445,7 @@ class HistoryDeserializer(sharable.SharableModelDeserializer, deletable.Purgable """ model_manager_class = HistoryManager - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): super().__init__(app) self.history_manager = self.manager @@ -482,7 +482,7 @@ class HistoriesService: and pydantic models to declare its parameters and return types. """ - def __init__(self, app: MininmalManagerApp, manager: HistoryManager, serializer: HistorySerializer): + def __init__(self, app: MinimalManagerApp, manager: HistoryManager, serializer: HistorySerializer): self.app = app self.manager = manager self.serializer = serializer diff --git a/lib/galaxy/managers/history_contents.py b/lib/galaxy/managers/history_contents.py index 50ee953f31e..d8d4db39973 100644 --- a/lib/galaxy/managers/history_contents.py +++ b/lib/galaxy/managers/history_contents.py @@ -32,7 +32,7 @@ from galaxy.managers import ( taggable, tools ) -from galaxy.structured_app import MininmalManagerApp +from galaxy.structured_app import MinimalManagerApp log = logging.getLogger(__name__) @@ -73,7 +73,7 @@ class HistoryContentsManager(containers.ContainerManagerMixin): ) default_order_by = 'hid' - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): self.app = app self.contained_manager = app[self.contained_class_manager_class] self.subcontainer_manager = app[self.subcontainer_class_manager_class] @@ -427,7 +427,7 @@ class HistoryContentsSerializer(base.ModelSerializer, deletable.PurgableSerializ """ model_manager_class = HistoryContentsManager - def __init__(self, app: MininmalManagerApp, **kwargs): + def __init__(self, app: MinimalManagerApp, **kwargs): super().__init__(app, **kwargs) self.default_view = 'summary' diff --git a/lib/galaxy/managers/lddas.py b/lib/galaxy/managers/lddas.py index fc0b18014c5..5f088b32078 100644 --- a/lib/galaxy/managers/lddas.py +++ b/lib/galaxy/managers/lddas.py @@ -3,7 +3,7 @@ import logging from galaxy import model, util from galaxy.managers import base as manager_base from galaxy.managers.datasets import DatasetAssociationManager -from galaxy.structured_app import MininmalManagerApp +from galaxy.structured_app import MinimalManagerApp log = logging.getLogger(__name__) @@ -14,7 +14,7 @@ class LDDAManager(DatasetAssociationManager): """ model_class = model.LibraryDatasetDatasetAssociation - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): """ Set up and initialize other managers needed by lddas. """ diff --git a/lib/galaxy/managers/metrics.py b/lib/galaxy/managers/metrics.py index 500acf60c1a..6ff09336451 100644 --- a/lib/galaxy/managers/metrics.py +++ b/lib/galaxy/managers/metrics.py @@ -13,7 +13,7 @@ from pydantic import ( Field, ) -from galaxy.app import MininmalManagerApp +from galaxy.app import MinimalManagerApp log = logging.getLogger(__name__) @@ -72,7 +72,7 @@ TimeSeriesTupleGenerator = Generator[TimeSeriesTuple, None, None] class MetricsManager: """Interface/service object shared by controllers for interacting with metrics.""" - def __init__(self, app: MininmalManagerApp) -> None: + def __init__(self, app: MinimalManagerApp) -> None: self._app = app #: set to true to send additional debugging info to the log self.debugging = True diff --git a/lib/galaxy/managers/pages.py b/lib/galaxy/managers/pages.py index 8b376dcebe1..18ef7921366 100644 --- a/lib/galaxy/managers/pages.py +++ b/lib/galaxy/managers/pages.py @@ -32,7 +32,7 @@ from galaxy.managers.markdown_util import ( ) from galaxy.model.item_attrs import UsesAnnotations from galaxy.schema.fields import EncodedDatabaseIdField -from galaxy.structured_app import MininmalManagerApp +from galaxy.structured_app import MinimalManagerApp from galaxy.util import unicodify from galaxy.util.sanitize_html import sanitize_html @@ -197,7 +197,7 @@ class PagesService: and pydantic models to declare its parameters and return types. """ - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): self.manager = PageManager(app) self.serializer = PageSerializer(app) self.shareable_service = sharable.ShareableService(self.manager, self.serializer) @@ -304,7 +304,7 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations): annotation_assoc = model.PageAnnotationAssociation rating_assoc = model.PageRatingAssociation - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): """ """ super().__init__(app) @@ -427,7 +427,7 @@ class PageSerializer(sharable.SharableModelSerializer): model_manager_class = PageManager SINGLE_CHAR_ABBR = 'p' - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): super().__init__(app) self.page_manager = PageManager(app) @@ -448,7 +448,7 @@ class PageDeserializer(sharable.SharableModelDeserializer): """ model_manager_class = PageManager - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): super().__init__(app) self.page_manager = self.manager diff --git a/lib/galaxy/managers/remote_files.py b/lib/galaxy/managers/remote_files.py index dabd2d0255d..9d40209fc83 100644 --- a/lib/galaxy/managers/remote_files.py +++ b/lib/galaxy/managers/remote_files.py @@ -12,7 +12,7 @@ from typing import ( from pydantic.tools import parse_obj_as from galaxy import exceptions -from galaxy.app import MininmalManagerApp +from galaxy.app import MinimalManagerApp from galaxy.files import ( ConfiguredFileSources, ProvidesUserFileSourcesUserContext, @@ -37,7 +37,7 @@ class RemoteFilesManager: Interface/service object for interacting with remote files. """ - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): self._app = app def index( diff --git a/lib/galaxy/managers/sharable.py b/lib/galaxy/managers/sharable.py index ad8c15f9791..5782895eae8 100644 --- a/lib/galaxy/managers/sharable.py +++ b/lib/galaxy/managers/sharable.py @@ -34,7 +34,7 @@ from galaxy.managers import ( ) from galaxy.model import UserShareAssociation from galaxy.schema.fields import EncodedDatabaseIdField -from galaxy.structured_app import MininmalManagerApp +from galaxy.structured_app import MinimalManagerApp from galaxy.util import ready_name_for_url log = logging.getLogger(__name__) @@ -51,7 +51,7 @@ class SharableModelManager(base.ModelManager, secured.OwnableManagerMixin, secur #: the single character abbreviation used in username_and_slug: e.g. 'h' for histories: u/user/h/slug SINGLE_CHAR_ABBR: Optional[str] = None - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): super().__init__(app) # user manager is needed to check access/ownership/admin self.user_manager = users.UserManager(app) diff --git a/lib/galaxy/managers/users.py b/lib/galaxy/managers/users.py index 42cd31f5541..cb9f4c67b73 100644 --- a/lib/galaxy/managers/users.py +++ b/lib/galaxy/managers/users.py @@ -30,7 +30,7 @@ from galaxy.security.validate_user_input import ( validate_password, validate_publicname ) -from galaxy.structured_app import BasicApp, MininmalManagerApp +from galaxy.structured_app import BasicApp, MinimalManagerApp from galaxy.util.hash_util import new_secure_hash from galaxy.web import url_for @@ -611,7 +611,7 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): class UserSerializer(base.ModelSerializer, deletable.PurgableSerializerMixin): model_manager_class = UserManager - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): """ Convert a User and associated data to a dictionary representation. """ diff --git a/lib/galaxy/managers/visualizations.py b/lib/galaxy/managers/visualizations.py index b49e2f11e1c..84f0d1fa1f9 100644 --- a/lib/galaxy/managers/visualizations.py +++ b/lib/galaxy/managers/visualizations.py @@ -10,7 +10,7 @@ from typing import Optional from galaxy import model from galaxy.managers import sharable from galaxy.schema.fields import EncodedDatabaseIdField -from galaxy.structured_app import MininmalManagerApp +from galaxy.structured_app import MinimalManagerApp log = logging.getLogger(__name__) @@ -43,7 +43,7 @@ class VisualizationSerializer(sharable.SharableModelSerializer): model_manager_class = VisualizationManager SINGLE_CHAR_ABBR = 'v' - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): super().__init__(app) self.visualization_manager = self.manager @@ -82,7 +82,7 @@ class VisualizationsService: and pydantic models to declare its parameters and return types. """ - def __init__(self, app: MininmalManagerApp, manager: VisualizationManager, serializer: VisualizationSerializer): + def __init__(self, app: MinimalManagerApp, manager: VisualizationManager, serializer: VisualizationSerializer): self.app = app self.manager = manager self.serializer = serializer diff --git a/lib/galaxy/managers/workflows.py b/lib/galaxy/managers/workflows.py index 95a280c3e57..5ee98861d8e 100644 --- a/lib/galaxy/managers/workflows.py +++ b/lib/galaxy/managers/workflows.py @@ -26,7 +26,7 @@ from galaxy import ( ) from galaxy.jobs.actions.post import ActionBox from galaxy.model.item_attrs import UsesAnnotations -from galaxy.structured_app import MininmalManagerApp +from galaxy.structured_app import MinimalManagerApp from galaxy.tools.parameters import ( params_to_incoming, visit_input_values @@ -69,7 +69,7 @@ class WorkflowsManager: the galaxy.workflow module. """ - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): self.app = app def get_stored_workflow(self, trans, workflow_id, by_stored_id=True): @@ -306,7 +306,7 @@ CreatedWorkflow = namedtuple("CreatedWorkflow", ["stored_workflow", "workflow", class WorkflowContentsManager(UsesAnnotations): - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): self.app = app self._resource_mapper_function = get_resource_mapper_function(app) diff --git a/lib/galaxy/structured_app.py b/lib/galaxy/structured_app.py index d179d98ec9f..1cb49fb8d39 100644 --- a/lib/galaxy/structured_app.py +++ b/lib/galaxy/structured_app.py @@ -77,7 +77,7 @@ class MinimalManagerApp(MinimalApp): pass -class StructuredApp(MininmalManagerApp): +class StructuredApp(MinimalManagerApp): """Interface defining typed description of the Galaxy UniverseApplication. Ideally nothing that depends on StructuredApp should require diff --git a/lib/galaxy/tools/cache.py b/lib/galaxy/tools/cache.py index b6eff3f9e10..351e654ba67 100644 --- a/lib/galaxy/tools/cache.py +++ b/lib/galaxy/tools/cache.py @@ -16,7 +16,7 @@ from sqlalchemy.orm import ( from sqlitedict import SqliteDict from galaxy.model.tool_shed_install import ToolShedRepository -from galaxy.structured_app import MininmalManagerApp +from galaxy.structured_app import MinimalManagerApp from galaxy.tools.toolbox.base import ToolConfRepository from galaxy.util import unicodify from galaxy.util.hash_util import md5_hash_file @@ -286,7 +286,7 @@ class ToolShedRepositoryCache: repositories: List[ToolShedRepository] repos_by_tuple: Dict[Tuple[str, str, str], List[ToolConfRepository]] - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): self.app = app # Contains ToolConfRepository objects created from shed_tool_conf.xml entries self.local_repositories = [] diff --git a/lib/galaxy/tools/data_manager/manager.py b/lib/galaxy/tools/data_manager/manager.py index 849a7fd4529..dd823b2e925 100644 --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -5,7 +5,7 @@ import os from typing import Dict from galaxy import util -from galaxy.structured_app import MininmalManagerApp +from galaxy.structured_app import MinimalManagerApp from galaxy.tools.data import TabularToolDataTable from galaxy.util.template import fill_template @@ -20,7 +20,7 @@ class DataManagers: data_managers: Dict[str, 'DataManager'] managed_data_tables: Dict[str, 'DataManager'] - def __init__(self, app: MininmalManagerApp, xml_filename=None): + def __init__(self, app: MinimalManagerApp, xml_filename=None): self.app = app self.data_managers = {} self.managed_data_tables = {} diff --git a/lib/galaxy/visualization/genomes.py b/lib/galaxy/visualization/genomes.py index 8738432c75a..3869a5178ec 100644 --- a/lib/galaxy/visualization/genomes.py +++ b/lib/galaxy/visualization/genomes.py @@ -11,7 +11,7 @@ from galaxy.exceptions import ( ObjectNotFound, ReferenceDataError, ) -from galaxy.structured_app import MininmalManagerApp +from galaxy.structured_app import MinimalManagerApp from galaxy.util.bunch import Bunch log = logging.getLogger(__name__) @@ -197,7 +197,7 @@ class Genomes: Provides information about available genome data and methods for manipulating that data. """ - def __init__(self, app: MininmalManagerApp): + def __init__(self, app: MinimalManagerApp): self.app = app # Create list of genomes from app.genome_builds self.genomes: Dict[str, Genome] = {} diff --git a/test/unit/unittest_utils/galaxy_mock.py b/test/unit/unittest_utils/galaxy_mock.py index a3779bded3a..02e44efe61a 100644 --- a/test/unit/unittest_utils/galaxy_mock.py +++ b/test/unit/unittest_utils/galaxy_mock.py @@ -21,7 +21,7 @@ from galaxy.model import mapping, tags from galaxy.model.base import SharedModelMapping from galaxy.model.mapping import GalaxyModelMapping from galaxy.security import idencoding -from galaxy.structured_app import BasicApp, MininmalManagerApp, StructuredApp +from galaxy.structured_app import BasicApp, MinimalManagerApp, StructuredApp from galaxy.tool_util.deps.containers import NullContainerFinder from galaxy.util import StructuredExecutionTimer from galaxy.util.bunch import Bunch @@ -67,7 +67,7 @@ class MockApp(di.Container): def __init__(self, config=None, **kwargs): super().__init__() self[BasicApp] = self - self[MininmalManagerApp] = self + self[MinimalManagerApp] = self self[StructuredApp] = self self.config = config or MockAppConfig(**kwargs) self.security = self.config.security From d300f9cca94da1c224cbb0ab6d49ee904cd5c548 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Mon, 29 Mar 2021 09:41:25 -0400 Subject: [PATCH 34/36] Allow usage of old queue_worker recalc task on logout so this doesn't block when celery isn't enabled --- lib/galaxy/queue_worker.py | 14 ++++++++++++++ lib/galaxy/webapps/galaxy/controllers/user.py | 7 +++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/queue_worker.py b/lib/galaxy/queue_worker.py index 53ee5376b66..d13b1666c50 100644 --- a/lib/galaxy/queue_worker.py +++ b/lib/galaxy/queue_worker.py @@ -224,6 +224,19 @@ def reload_sanitize_allowlist(app): app.config.reload_sanitize_allowlist() +def recalculate_user_disk_usage(app, **kwargs): + user_id = kwargs.get('user_id', None) + sa_session = app.model.context + if user_id: + user = sa_session.query(app.model.User).get(app.security.decode_id(user_id)) + if user: + user.calculate_and_set_disk_usage() + else: + log.error("Recalculate user disk usage task failed, user %s not found" % user_id) + else: + log.error("Recalculate user disk usage task received without user_id.") + + def reload_tool_data_tables(app, **kwargs): path = kwargs.get('path') table_name = kwargs.get('table_name') @@ -308,6 +321,7 @@ control_message_to_task = { 'reload_job_rules': reload_job_rules, 'admin_job_lock': admin_job_lock, 'reload_sanitize_allowlist': reload_sanitize_allowlist, + 'recalculate_user_disk_usage': recalculate_user_disk_usage, 'rebuild_toolbox_search_index': rebuild_toolbox_search_index, 'reconfigure_watcher': reconfigure_watcher, 'reload_tour': reload_tour, diff --git a/lib/galaxy/webapps/galaxy/controllers/user.py b/lib/galaxy/webapps/galaxy/controllers/user.py index a3c042cb6f4..ffba90b02ab 100644 --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -13,9 +13,9 @@ from galaxy import ( util, web ) -from galaxy.celery.tasks import recalculate_user_disk_usage from galaxy.exceptions import Conflict from galaxy.managers import users +from galaxy.queue_worker import send_local_control_task from galaxy.security.validate_user_input import ( validate_email, validate_publicname @@ -225,9 +225,12 @@ class User(BaseUIController, UsesFormDefinitionsMixin, CreatesApiKeysMixin): if trans.app.config.enable_celery_tasks: # Queue a quota recalculation (async) task -- this takes a # while sometimes, so we don't want to block on logout. + from galaxy.celery.tasks import recalculate_user_disk_usage recalculate_user_disk_usage.delay(user_id=trans.user.id) else: - trans.user.calculate_and_set_disk_usage() + send_local_control_task(trans.app, + "recalculate_user_disk_usage", + kwargs={"user_id": trans.security.encode_id(trans.user.id)}) # Since logging an event requires a session, we'll log prior to ending the session trans.log_event("User logged out") trans.handle_user_logout(logout_all=logout_all) From 024ccf6beb00755e34822752816128cd16637c8c Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Mon, 29 Mar 2021 09:56:56 -0400 Subject: [PATCH 35/36] Reapply changes dropped in rebase shuffle --- lib/galaxy/celery/tasks.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/celery/tasks.py b/lib/galaxy/celery/tasks.py index 01068b1a752..5f0f02e0a22 100644 --- a/lib/galaxy/celery/tasks.py +++ b/lib/galaxy/celery/tasks.py @@ -5,6 +5,7 @@ from sqlalchemy.orm.scoping import ( from galaxy.celery import celery_app from galaxy.managers.hdas import HDAManager +from galaxy.managers.lddas import LDDAManager from galaxy.model import User from galaxy.util.custom_logging import get_logger from . import get_galaxy_app @@ -19,7 +20,7 @@ def galaxy_task(func): return func -@celery_app.task +@celery_app.task(ignore_result=True) @galaxy_task def recalculate_user_disk_usage(session: scoped_session, user_id=None): if user_id: @@ -33,8 +34,18 @@ def recalculate_user_disk_usage(session: scoped_session, user_id=None): log.error("Recalculate user disk usage task received without user_id.") -@celery_app.task +@celery_app.task(ignore_result=True) @galaxy_task def purge_hda(hda_manager: HDAManager, hda_id): hda = hda_manager.by_id(hda_id) hda_manager._purge(hda) + + +@celery_app.task +@galaxy_task +def set_metadata(hda_manager: HDAManager, ldda_manager: LDDAManager, dataset_id, model_class='HistoryDatasetAssociation'): + if model_class == 'HistoryDatasetAssociation': + dataset = hda_manager.by_id(dataset_id) + elif model_class == 'LibraryDatasetDatasetAssociation': + dataset = ldda_manager.by_id(dataset_id) + dataset.datatype.set_meta(dataset) From 537495a4e3481c416de348ab6e2159616793d72e Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 29 Mar 2021 16:15:19 +0200 Subject: [PATCH 36/36] Do local import --- lib/galaxy/managers/hdas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/managers/hdas.py b/lib/galaxy/managers/hdas.py index d87d9f55f1e..dc6c2b2aeba 100644 --- a/lib/galaxy/managers/hdas.py +++ b/lib/galaxy/managers/hdas.py @@ -13,7 +13,6 @@ from galaxy import ( exceptions, model ) -from galaxy.celery.tasks import purge_hda from galaxy.managers import ( annotatable, datasets, @@ -137,6 +136,7 @@ class HDAManager(datasets.DatasetAssociationManager, # .... deletion and purging def purge(self, hda, flush=True): if self.app.config.enable_celery_tasks: + from galaxy.celery.tasks import purge_hda purge_hda.delay(hda_id=hda.id) else: self._purge(hda, flush=flush)