From a20919edaa1bad07a44e413327cf94ee19b1b4b9 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 22 Feb 2021 11:12:36 +0100 Subject: [PATCH 1/2] Make hash selection mandatory --- tools/filters/secure_hash_message_digest.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/filters/secure_hash_message_digest.xml b/tools/filters/secure_hash_message_digest.xml index 50554c7181d..329f3635351 100644 --- a/tools/filters/secure_hash_message_digest.xml +++ b/tools/filters/secure_hash_message_digest.xml @@ -13,14 +13,13 @@ - + From dc961c8f69b69553c8256805c154933ea63035b2 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 22 Feb 2021 18:41:35 +0100 Subject: [PATCH 2/2] Wrap core of workflow monitor thread in try/except Should prevent an exception like this: ``` galaxy.workflow.scheduling_manager DEBUG 2021-02-21 15:59:11,789 Attempting to schedule workflow invocation [(535353,)] Exception in thread WorkflowRequestMonitor.monitor_thread: Traceback (most recent call last): File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 2336, in _wrap_pool_connect return fn() File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 364, in conne ct return _ConnectionFairy._checkout(self) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 778, in _checkout fairy = _ConnectionRecord.checkout(pool) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 500, in checkout rec._checkin_failed(err) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__ with_traceback=exc_tb, File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 182, in raise_ raise exception File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 497, in checkout dbapi_connection = rec.get_connection() File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 610, in get_connection self.__connect() File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 661, in __connect pool.logger.debug("Error on connect(): %s", e) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__ with_traceback=exc_tb, File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 182, in raise_ raise exception File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 656, in __connect connection = pool._invoke_creator(self) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/engine/strategies.py", line 114, in connect return dialect.connect(*cargs, **cparams) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 493, in connect return self.dbapi.connect(*cargs, **cparams) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/psycopg2/__init__.py", line 126, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: FATAL: the database system is in recovery mode The above exception was the direct cause of the following exception: Traceback (most recent call last): File "/cvmfs/main.galaxyproject.org/deps/_conda/envs/_galaxy_/lib/python3.6/threading.py", line 916, in _bootstrap_inner self.run() File "/cvmfs/main.galaxyproject.org/deps/_conda/envs/_galaxy_/lib/python3.6/threading.py", line 864, in run self._target(*self._args, **self._kwargs) File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/workflow/scheduling_manager.py", line 297, in __monitor self.__schedule(workflow_scheduler_id, workflow_scheduler) File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/workflow/scheduling_manager.py", line 305, in __schedule self.__attempt_schedule(invocation_id, workflow_scheduler) File "/cvmfs/main.galaxyproject.org/galaxy/lib/galaxy/workflow/scheduling_manager.py", line 311, in __attempt_schedule workflow_invocation = sa_session.query(model.WorkflowInvocation).get(invocation_id) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 1018, in get return self._get_impl(ident, loading.load_on_pk_identity) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 1135, in _get_impl return db_load_fn(self, primary_key_identity) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/loading.py", line 286, in load_on_pk_identity return q.one() File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3490, in one ret = self.one_or_none() File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3459, in one_or_none ret = list(self) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3535, in __it er__ return self._execute_and_instances(context) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3557, in _execute_and_instances querycontext, self._connection_from_session, close_with_result=True File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3572, in _get_bind_args mapper=self._bind_mapper(), clause=querycontext.statement, **kw File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3550, in _connection_from_session conn = self.session.connection(**kw) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 1141, in connection execution_options=execution_options, File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 1150, in _connection_for_bind conn = engine._contextual_connect(**kw) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 2302, in _contextual_connect self._wrap_pool_connect(self.pool.connect, None), File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 2340, in _wrap_pool_connect e, dialect, self File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1584, in _h andle_dbapi_exception_noconnection sqlalchemy_exception, with_traceback=exc_info[2], from_=e File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 182, in raise_ raise exception File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 2336, in _wrap_pool_connect return fn() File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 364, in connect return _ConnectionFairy._checkout(self) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 778, in _checkout fairy = _ConnectionRecord.checkout(pool) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 500, in checkout rec._checkin_failed(err) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__ with_traceback=exc_tb, File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 182, in raise_ raise exception File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 497, in check out dbapi_connection = rec.get_connection() File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 610, in get_connection self.__connect() File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 661, in __connect pool.logger.debug("Error on connect(): %s", e) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__ with_traceback=exc_tb, File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 182, in raise_ raise exception File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py", line 656, in __connect connection = pool._invoke_creator(self) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/engine/strategies.py", line 114, in connect return dialect.connect(*cargs, **cparams) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 493, in connect return self.dbapi.connect(*cargs, **cparams) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.6/site-packages/psycopg2/__init__.py", line 126, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) FATAL: the database system is in recovery mode ``` to stop the monitor thread. --- lib/galaxy/workflow/scheduling_manager.py | 25 +++++++++++++---------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/workflow/scheduling_manager.py b/lib/galaxy/workflow/scheduling_manager.py index 46764ec3815..a51b06f95cc 100644 --- a/lib/galaxy/workflow/scheduling_manager.py +++ b/lib/galaxy/workflow/scheduling_manager.py @@ -283,19 +283,22 @@ class WorkflowRequestMonitor(Monitors): def __monitor(self): to_monitor = self.workflow_scheduling_manager.active_workflow_schedulers while self.monitor_running: - if self.invocation_grabber: - self.invocation_grabber.grab_unhandled_items() + try: + if self.invocation_grabber: + self.invocation_grabber.grab_unhandled_items() - monitor_step_timer = self.app.execution_timer_factory.get_timer( - 'internal.galaxy.workflows.scheduling_manager.monitor_step', - 'Workflow scheduling manager monitor step complete.' - ) - for workflow_scheduler_id, workflow_scheduler in to_monitor.items(): - if not self.monitor_running: - return + monitor_step_timer = self.app.execution_timer_factory.get_timer( + 'internal.galaxy.workflows.scheduling_manager.monitor_step', + 'Workflow scheduling manager monitor step complete.' + ) + for workflow_scheduler_id, workflow_scheduler in to_monitor.items(): + if not self.monitor_running: + return - self.__schedule(workflow_scheduler_id, workflow_scheduler) - log.trace(monitor_step_timer.to_str()) + self.__schedule(workflow_scheduler_id, workflow_scheduler) + log.trace(monitor_step_timer.to_str()) + except Exception: + log.exception('An exception occured scheduling while scheduling workflows') self._monitor_sleep(1) def __schedule(self, workflow_scheduler_id, workflow_scheduler):