From fc7f367c7d647938c22c2107fb0e0c64304629f8 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Wed, 20 Oct 2021 12:55:24 +0200 Subject: [PATCH 1/5] rephrase and fix typos --- lib/galaxy/workflow/modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 9528dc7a7ba..5cc2d44c3e1 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -953,10 +953,10 @@ class InputParameterModule(WorkflowModule): if restrictions_list is None: restrictions_list = [] restriction_values = self._parameter_option_def_to_tool_form_str(restrictions_list) - restrictions_source = dict(name="restrictions", label="Restriction Values", value=restriction_values, help="Comman-separated list of potential all values") + restrictions_source = dict(name="restrictions", label="Restricted Values", value=restriction_values, help="Comma-separated list of all permitted values") restrictions = TextToolParameter(None, restrictions_source) - suggestions_source = dict(name="suggestions", label="Suggestion Values", value=restriction_values, help="Comman-separated list of some potential values") + suggestions_source = dict(name="suggestions", label="Suggested Values", value=restriction_values, help="Comma-separated list of some potential values") suggestions = TextToolParameter(None, suggestions_source) when_restrict_static_restrictions.inputs["restrictions"] = restrictions From 29e09017942e36ca63aa9427983a7bc8c33b2bf8 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 21 Oct 2021 07:37:54 -0400 Subject: [PATCH 2/5] Prefer model class when it exists (assuming it is more correct/refined logic) over column. This fixes history sort to work correctly in grids. --- lib/galaxy/web/framework/helpers/grids.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/web/framework/helpers/grids.py b/lib/galaxy/web/framework/helpers/grids.py index 7566e7f635f..be220f00577 100644 --- a/lib/galaxy/web/framework/helpers/grids.py +++ b/lib/galaxy/web/framework/helpers/grids.py @@ -76,9 +76,9 @@ class GridColumn: """Sort query using this column.""" if column_name is None: column_name = self.key - column = self.model_class.table.c.get(column_name) + column = getattr(self.model_class, column_name) if column is None: - column = getattr(self.model_class, column_name) + column = self.model_class.table.c.get(column_name) if ascending: query = query.order_by(column.asc()) else: From 2dd0eed46b8beefdc650493948dbbcfe0eae01a4 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 21 Oct 2021 12:28:26 -0400 Subject: [PATCH 3/5] Fix several 'to to' redundant word typos. --- client/src/components/HistoryExport/ToLink.vue | 2 +- lib/galaxy/tool_util/xsd/galaxy.xsd | 2 +- lib/galaxy/webapps/base/webapp.py | 4 ++-- scripts/release.sh | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/client/src/components/HistoryExport/ToLink.vue b/client/src/components/HistoryExport/ToLink.vue index 9ede813b4c9..173e372476d 100644 --- a/client/src/components/HistoryExport/ToLink.vue +++ b/client/src/components/HistoryExport/ToLink.vue @@ -35,7 +35,7 @@

Click here to to generate a new archive for this history.Click here to generate a new archive for this history.

diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 5967baa82fe..614dd20d94a 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -5520,7 +5520,7 @@ Examples are included in the test tools directory including: - '$__tool_directory__/'`` in the tag content. If this attribute is not specified, the tag should contain a Bash command calling executable(s) available in the ``$PATH``, as modified after loading the requirements.]]> + '$__tool_directory__/'`` in the tag content. If this attribute is not specified, the tag should contain a Bash command calling executable(s) available in the ``$PATH``, as modified after loading the requirements.]]> diff --git a/lib/galaxy/webapps/base/webapp.py b/lib/galaxy/webapps/base/webapp.py index 5aca1611849..7d9d54b7159 100644 --- a/lib/galaxy/webapps/base/webapp.py +++ b/lib/galaxy/webapps/base/webapp.py @@ -456,8 +456,8 @@ class GalaxyWebTransaction(base.DefaultWebTransaction, context.ProvidesHistoryCo # We'll end up creating a new galaxy_session session_key = None # If remote user is in use it can invalidate the session and in some - # cases won't have a cookie set above, so we need to to check some - # things now. + # cases won't have a cookie set above, so we need to check some things + # now. if self.app.config.use_remote_user: remote_user_email = self.environ.get(self.app.config.remote_user_header, None) if galaxy_session: diff --git a/scripts/release.sh b/scripts/release.sh index 4b2a984f6f0..81150e4ba29 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -298,7 +298,7 @@ function perform_stable_merge() { log_exec git merge -m "Merge branch 'release_${RELEASE_CURR}' into '${STABLE_BRANCH}'" "__release_${RELEASE_CURR}" PUSH_BRANCHES+=("__stable:${STABLE_BRANCH}") else - log "Release '${RELEASE_CURR}' < stable branch release '${stable}', skipping merge to to '${STABLE_BRANCH}'" + log "Release '${RELEASE_CURR}' < stable branch release '${stable}', skipping merge to '${STABLE_BRANCH}'" fi git checkout "$branch_curr" } From 2fd861bf8966f4e0f00cfb46ab67f1ecc543146e Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 21 Oct 2021 18:04:10 +0200 Subject: [PATCH 4/5] Improve error handling in DirectoryUriToolParameter validation Avoids 500 status code if the parameter has a wrong value --- lib/galaxy/tools/parameters/basic.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index e26ae8a013e..d085eeb73d4 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -2419,7 +2419,10 @@ class DirectoryUriToolParameter(SimpleTextToolParameter): super().validate(value, trans=trans) if not value: return # value is not set yet, do not validate - file_source = trans.app.file_sources.get_file_source_path(value).file_source + file_source_path = trans.app.file_sources.get_file_source_path(value) + file_source = file_source_path.file_source + if file_source is None: + raise ParameterValueError(f"'{value}' is not a valid file source uri.", self.name) user_context = ProvidesUserFileSourcesUserContext(trans) user_has_access = file_source.user_has_access(user_context) if not user_has_access: From 2a335c981b2eecca6a48754c5a137f09af8db5cc Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 21 Oct 2021 23:40:55 +0200 Subject: [PATCH 5/5] Fix double enqueuing of upload jobs The canonical place to call enqueue is in https://github.com/galaxyproject/galaxy/blob/a5bb3e92667b96ef802b378f17788b253a5c0bc6/lib/galaxy/tools/execute.py#L127. In the process of enqueue the db-skip-locked assignment method will set the handler name to the default tag. If you do this more than once the job might be picked up by compatible handlers. Fixes https://github.com/galaxyproject/galaxy/issues/11335 --- lib/galaxy/actions/library.py | 3 +-- lib/galaxy/datatypes/data.py | 7 ++++--- lib/galaxy/managers/datasets.py | 6 ++++-- lib/galaxy/tools/__init__.py | 3 ++- lib/galaxy/tools/actions/__init__.py | 3 --- lib/galaxy/tools/actions/history_imp_exp.py | 5 ----- lib/galaxy/tools/actions/metadata.py | 5 ----- lib/galaxy/tools/actions/upload_common.py | 14 +------------- lib/galaxy/webapps/base/controller.py | 6 ++++-- lib/galaxy/webapps/galaxy/api/library_datasets.py | 3 +-- lib/galaxy/webapps/galaxy/controllers/async.py | 3 ++- lib/galaxy/webapps/galaxy/controllers/dataset.py | 3 ++- 12 files changed, 21 insertions(+), 40 deletions(-) diff --git a/lib/galaxy/actions/library.py b/lib/galaxy/actions/library.py index bd3d6f009d9..1fe6af270da 100644 --- a/lib/galaxy/actions/library.py +++ b/lib/galaxy/actions/library.py @@ -126,8 +126,7 @@ class LibraryActions: job_params['link_data_only'] = json.dumps(kwd.get('link_data_only', 'copy_files')) job_params['uuid'] = json.dumps(kwd.get('uuid', None)) job, output = upload_common.create_job(trans, tool_params, tool, json_file_path, data_list, folder=library_bunch.folder, job_params=job_params) - trans.sa_session.add(job) - trans.sa_session.flush() + trans.app.job_manager.enqueue(job, tool=tool) return output def _get_server_dir_uploaded_datasets(self, trans, params, full_dir, import_dir_desc, library_bunch, response_code, message): diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 8bf8060535c..2103d338169 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -637,14 +637,15 @@ class Data(metaclass=DataMeta): params[input_name] = original_dataset # Run converter, job is dispatched through Queue - converted_dataset = converter.execute(trans, incoming=params, set_output_hid=visible, history=history)[1] + job, converted_datasets, *_ = converter.execute(trans, incoming=params, set_output_hid=visible, history=history) + trans.app.job_manager.enqueue(job, tool=converter) if len(params) > 0: trans.log_event("Converter params: %s" % (str(params)), tool_id=converter.id) if not visible: - for value in converted_dataset.values(): + for value in converted_datasets.values(): value.visible = False if return_output: - return converted_dataset + return converted_datasets return f"The file conversion of {converter.name} on data {original_dataset.hid} has been added to the Queue." # We need to clear associated files before we set metadata diff --git a/lib/galaxy/managers/datasets.py b/lib/galaxy/managers/datasets.py index 483e13004fc..379a5a15612 100644 --- a/lib/galaxy/managers/datasets.py +++ b/lib/galaxy/managers/datasets.py @@ -374,9 +374,10 @@ class DatasetAssociationManager(base.ModelManager, if spec.get('default'): setattr(data.metadata, name, spec.unwrap(spec.get('default'))) - self.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( + job, *_ = self.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( self.app.datatypes_registry.set_external_metadata_tool, trans, incoming={'input1': data, 'validate': validate}, overwrite=overwrite) + self.app.job_manager.enqueue(job, tool=self.app.datatypes_registry.set_external_metadata_tool) def update_permissions(self, trans, dataset_assoc, **kwd): action = kwd.get('action', 'set_permissions') @@ -691,7 +692,8 @@ class DatasetAssociationDeserializer(base.ModelDeserializer, deletable.PurgableD sa_session = self.app.model.context sa_session.flush() trans = context.get("trans") - self.app.datatypes_registry.set_external_metadata_tool.tool_action.execute(self.app.datatypes_registry.set_external_metadata_tool, trans, incoming={'input1': item}, overwrite=False) # overwrite is False as per existing behavior + job, *_ = self.app.datatypes_registry.set_external_metadata_tool.tool_action.execute(self.app.datatypes_registry.set_external_metadata_tool, trans, incoming={'input1': item}, overwrite=False) # overwrite is False as per existing behavior + trans.app.job_manager.enqueue(job, tool=trans.app.datatypes_registry.set_external_metadata_tool) return item.datatype diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 6de749ae0ab..50174d26701 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2606,10 +2606,11 @@ class SetMetadataTool(Tool): def regenerate_imported_metadata_if_needed(self, hda, history, job): if len(hda.metadata_file_types) > 0: - self.tool_action.execute_via_app( + job, *_ = self.tool_action.execute_via_app( self, self.app, job.session_id, history.id, job.user, incoming={'input1': hda}, overwrite=False ) + self.app.job_manager.enqueue(job=job, tool=self) def exec_after_process(self, app, inp_data, out_data, param_dict, job=None, **kwds): working_directory = app.object_store.get_filename( diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 977a9da76e9..32717bc2f3a 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -600,9 +600,6 @@ class DefaultToolAction: trans.sa_session.flush() log.info(f"Flushed transaction for job {job.log_str()} {job_flush_timer}") - # Dispatch to a job handler. enqueue() is responsible for flushing the job - app.job_manager.enqueue(job, tool=tool) - trans.log_event("Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id) return job, out_data, history def _remap_job_on_rerun(self, trans, galaxy_session, rerun_remap_job_id, current_job, out_data): diff --git a/lib/galaxy/tools/actions/history_imp_exp.py b/lib/galaxy/tools/actions/history_imp_exp.py index 8e265ae1092..c0a7afd11fd 100644 --- a/lib/galaxy/tools/actions/history_imp_exp.py +++ b/lib/galaxy/tools/actions/history_imp_exp.py @@ -67,11 +67,6 @@ class ImportHistoryToolAction(ToolAction): job.add_parameter(name, value) job.state = start_job_state # job inputs have been configured, restore initial job state - - # Queue the job for execution - trans.app.job_manager.enqueue(job, tool=tool) - trans.log_event("Added import history job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id) - return job, {} diff --git a/lib/galaxy/tools/actions/metadata.py b/lib/galaxy/tools/actions/metadata.py index 33647c5104c..acd02da1c76 100644 --- a/lib/galaxy/tools/actions/metadata.py +++ b/lib/galaxy/tools/actions/metadata.py @@ -120,11 +120,6 @@ class SetMetadataToolAction(ToolAction): job.state = start_job_state # job inputs have been configured, restore initial job state sa_session.flush() - # Queue the job for execution - app.job_manager.enqueue(job, tool=tool) - # FIXME: need to add event logging to app and log events there rather than trans. - # trans.log_event( "Added set external metadata job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id ) - # clear e.g. converted files dataset.datatype.before_setting_metadata(dataset) diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index 9abb79fa14a..df01ddabe86 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -395,6 +395,7 @@ def create_job(trans, params, tool, json_file_path, outputs, folder=None, histor Create the upload job. """ job = trans.app.model.Job() + trans.sa_session.add(job) job.galaxy_version = trans.app.config.version_major galaxy_session = trans.get_galaxy_session() if type(galaxy_session) == trans.model.GalaxySession: @@ -410,16 +411,10 @@ def create_job(trans, params, tool, json_file_path, outputs, folder=None, histor job.tool_id = tool.id job.tool_version = tool.version job.dynamic_tool = tool.dynamic_tool - job.set_state(job.states.UPLOAD) - trans.sa_session.add(job) - trans.sa_session.flush() - log.info('tool %s created job id %d' % (tool.id, job.id)) - trans.log_event('created job id %d' % job.id, tool_id=tool.id) for name, value in tool.params_to_strings(params, trans.app).items(): job.add_parameter(name, value) job.add_parameter('paramfile', dumps(json_file_path)) - object_store_id = None for i, output_object in enumerate(outputs): output_name = "output%i" % i if hasattr(output_object, "collection"): @@ -432,18 +427,11 @@ def create_job(trans, params, tool, json_file_path, outputs, folder=None, histor else: job.add_output_dataset(output_name, dataset) - trans.sa_session.add(output_object) - - job.object_store_id = object_store_id job.set_state(job.states.NEW) if job_params: for name, value in job_params.items(): job.add_parameter(name, value) - trans.sa_session.add(job) - # Queue the job for execution - trans.app.job_manager.enqueue(job, tool=tool) - trans.log_event("Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id) output = {} for i, v in enumerate(outputs): if not hasattr(output_object, "collection_type"): diff --git a/lib/galaxy/webapps/base/controller.py b/lib/galaxy/webapps/base/controller.py index 13a2be6c2df..697de41db07 100644 --- a/lib/galaxy/webapps/base/controller.py +++ b/lib/galaxy/webapps/base/controller.py @@ -436,7 +436,8 @@ class ExportsHistoryMixin: # Run job to do export. history_exp_tool = trans.app.toolbox.get_tool(export_tool_id) - job, _ = history_exp_tool.execute(trans, incoming=params, history=history, set_output_hid=True) + job, *_ = history_exp_tool.execute(trans, incoming=params, history=history, set_output_hid=True) + trans.app.job_manager.enqueue(job, tool=history_exp_tool) return job @@ -446,7 +447,8 @@ class ImportsHistoryMixin: # Run job to do import. history_imp_tool = trans.app.toolbox.get_tool('__IMPORT_HISTORY__') incoming = {'__ARCHIVE_SOURCE__': archive_source, '__ARCHIVE_TYPE__': archive_type} - job, _ = history_imp_tool.execute(trans, incoming=incoming) + job, *_ = history_imp_tool.execute(trans, incoming=incoming) + trans.app.job_manager.enqueue(job, tool=history_imp_tool) return job diff --git a/lib/galaxy/webapps/galaxy/api/library_datasets.py b/lib/galaxy/webapps/galaxy/api/library_datasets.py index 624b9744614..28a8b27fb7c 100644 --- a/lib/galaxy/webapps/galaxy/api/library_datasets.py +++ b/lib/galaxy/webapps/galaxy/api/library_datasets.py @@ -519,8 +519,7 @@ class LibraryDatasetsController(BaseGalaxyAPIController, UsesVisualizationMixin, job_params['link_data_only'] = dumps(kwd.get('link_data_only', 'copy_files')) job_params['uuid'] = dumps(kwd.get('uuid', None)) job, output = upload_common.create_job(trans, tool_params, tool, json_file_path, data_list, folder=folder, job_params=job_params) - trans.sa_session.add(job) - trans.sa_session.flush() + trans.app.job_manager.enqueue(job, tool=tool) job_dict = job.to_dict() job_dict['id'] = trans.security.encode_id(job_dict['id']) return job_dict diff --git a/lib/galaxy/webapps/galaxy/controllers/async.py b/lib/galaxy/webapps/galaxy/controllers/async.py index 10d82a006c9..d4b70419ed1 100644 --- a/lib/galaxy/webapps/galaxy/controllers/async.py +++ b/lib/galaxy/webapps/galaxy/controllers/async.py @@ -89,7 +89,8 @@ class ASync(BaseUIController): raise Exception("Error: ToolOutput object not found") original_history = trans.sa_session.query(trans.app.model.History).get(data.history_id) - tool.execute(trans, incoming=params, history=original_history) + job, *_ = tool.execute(trans, incoming=params, history=original_history) + trans.app.job_manager.enqueue(job, tool=tool) else: log.debug('async error -> %s' % STATUS) trans.log_event('Async error -> %s' % STATUS) diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py index 2335b14c4b5..f008450547c 100644 --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -377,9 +377,10 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE datatype = sniff.guess_ext(path, trans.app.datatypes_registry.sniff_order, is_binary=is_binary) trans.app.datatypes_registry.change_datatype(data, datatype) trans.sa_session.flush() - trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( + job, *_ = trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, trans, incoming={'input1': data}, overwrite=False) # overwrite is False as per existing behavior + trans.app.job_manager.enqueue(job, tool=trans.app.datatypes_registry.set_external_metadata_tool) message = 'Detection was finished and changed the datatype to %s.' % datatype else: return self.message_exception(trans, 'Changing datatype "%s" is not allowed.' % (data.extension))