From 8d8f6cf84b493e5d85e8be4b356a33472106a000 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 2 Feb 2022 13:19:15 +0100 Subject: [PATCH 001/652] Fix construction of SelectToolParameter default value --- lib/galaxy/workflow/modules.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 586ae579358..79dd9b37db4 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -828,7 +828,7 @@ class InputParameterModule(WorkflowModule): name = "Input parameter" default_parameter_type = "text" default_optional = False - default_default_value = '' + default_default_value = None parameter_type = default_parameter_type optional = default_optional default_value = default_default_value @@ -1015,6 +1015,7 @@ class InputParameterModule(WorkflowModule): parameter_def = self._parse_state_into_dict() parameter_type = parameter_def["parameter_type"] optional = parameter_def["optional"] + default_value = parameter_def.get("default", self.default_default_value) if parameter_type not in ["text", "boolean", "integer", "float", "color"]: raise ValueError("Invalid parameter type for workflow parameters encountered.") @@ -1044,7 +1045,7 @@ class InputParameterModule(WorkflowModule): options = None if static_options and len(static_options) == 1: # If we are connected to a single option, just use it as is so order is preserved cleanly and such. - options = [{"label": o[0], "value": o[1]} for o in static_options[0]] + options = [{"label": o[0], "value": o[1], "selected": default_value and o[1] == default_value} for o in static_options[0]] elif static_options: # Intersection based on values of multiple option connections. intxn_vals = set.intersection(*({option[1] for option in options} for options in static_options)) @@ -1052,7 +1053,7 @@ class InputParameterModule(WorkflowModule): d = defaultdict(set) # Collapse labels with same values for label, value, _ in intxn_opts: d[value].add(label) - options = [{"label": ', '.join(label), "value": value, "selected": False} for value, label in d.items()] + options = [{"label": ', '.join(label), "value": value, "selected": default_value and value == default_value} for value, label in d.items()] if options is not None: parameter_kwds["options"] = options @@ -1089,8 +1090,10 @@ class InputParameterModule(WorkflowModule): parameter_class = parameter_types[client_parameter_type] if optional: - default_value = parameter_def.get("default", self.default_default_value) - parameter_kwds["value"] = default_value + if client_parameter_type == "select": + parameter_kwds['selected'] = default_value + else: + parameter_kwds["value"] = default_value if parameter_type == 'boolean': parameter_kwds['checked'] = default_value From 6b65de8f3859ed94ab19289a8c6440a2c4321454 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 2 Feb 2022 15:35:25 +0100 Subject: [PATCH 002/652] Add selenium test for restrictOnConnections --- lib/galaxy/selenium/navigation.yml | 3 +++ lib/galaxy_test/selenium/test_workflow_run.py | 26 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml index ebc97f3e2b7..debfd06a3d9 100644 --- a/lib/galaxy/selenium/navigation.yml +++ b/lib/galaxy/selenium/navigation.yml @@ -526,6 +526,9 @@ workflow_run: subworkflow_step_icon: ".portlet-title-icon.fa-sitemap" run_workflow: "#run-workflow" validation_error: ".validation-error" + input_select_field: + type: xpath + selector: '//div[@data-label="${label}"]//span[@class="select2-chosen"]' workflow_editor: diff --git a/lib/galaxy_test/selenium/test_workflow_run.py b/lib/galaxy_test/selenium/test_workflow_run.py index c8197651146..e955fc829ea 100644 --- a/lib/galaxy_test/selenium/test_workflow_run.py +++ b/lib/galaxy_test/selenium/test_workflow_run.py @@ -188,6 +188,32 @@ steps: content = self.dataset_populator.get_history_dataset_content(history_id, hid=7) self.assertEqual("10.0\n30.0\n20.0\n40.0\n", content) + @selenium_test + @managed_history + def test_execution_with_text_default_value_connected_to_restricted_select(self): + self.open_in_workflow_run(""" +class: GalaxyWorkflow +inputs: + text_param: + optional: true + default: ex2 + restrictOnConnections: true + type: text +steps: + multi_select: + tool_id: multi_select + in: + select_ex: + source: text_param +""") + element = self.components.workflow_run.input_select_field(label="text_param").wait_for_present() + assert element.text == 'Ex2' + self.workflow_run_submit() + history_id = self.current_history_id() + self.workflow_populator.wait_for_history_workflows(history_id, expected_invocation_count=1) + content = self.dataset_populator.get_history_dataset_content(history_id, hid=1) + assert content == 'ex2' + @selenium_test @managed_history def test_execution_with_rules(self): From 51d728b4c2cb437f06e280cd4343984b8ec9bbd2 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 4 Feb 2022 14:02:52 +0100 Subject: [PATCH 003/652] Move option restriction to separate method --- lib/galaxy/model/__init__.py | 2 +- lib/galaxy/workflow/modules.py | 75 ++++++++++++++++++---------------- 2 files changed, 41 insertions(+), 36 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 6e7b6ff84bb..6af18041e32 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6695,7 +6695,7 @@ class WorkflowStepConnection(Base, RepresentById): return self.input_step_input.name @property - def input_step(self): + def input_step(self) -> Union[WorkflowStep, None]: return self.input_step_input and self.input_step_input.workflow_step @property diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 79dd9b37db4..262c9bf961e 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -5,7 +5,7 @@ import json import logging import re from collections import defaultdict -from typing import Any, cast, Dict, List, Optional, Union +from typing import Any, cast, Dict, Iterable, List, Optional, Union import packaging.version from typing_extensions import TypedDict @@ -17,7 +17,7 @@ from galaxy import ( ) from galaxy.exceptions import ToolMissingException from galaxy.job_execution.actions.post import ActionBox -from galaxy.model import PostJobAction, Workflow +from galaxy.model import PostJobAction, Workflow, WorkflowStepConnection from galaxy.model.dataset_collections import matching from galaxy.tool_util.parser.output_objects import ToolExpressionOutput from galaxy.tools import ( @@ -1011,7 +1011,38 @@ class InputParameterModule(WorkflowModule): parameter_type_cond.cases = cases return {"parameter_definition": parameter_type_cond} - def get_runtime_inputs(self, connections=None, **kwds): + def restrict_options(self, connections: Iterable[WorkflowStepConnection], default_value): + try: + static_options = [] + # Retrieve possible runtime options for 'select' type inputs + for connection in connections: + # Well this isn't a great assumption... + module = connection.input_step.module # type: ignore[union-attr] + tool_inputs = module.tool.inputs # may not be set, but we're catching the Exception below. + + def callback(input, prefixed_name, context, **kwargs): + if prefixed_name == connection.input_name and hasattr(input, 'get_options'): + static_options.append(input.get_options(self.trans, {})) + visit_input_values(tool_inputs, module.state.inputs, callback) + + options = None + if static_options and len(static_options) == 1: + # If we are connected to a single option, just use it as is so order is preserved cleanly and such. + options = [{"label": o[0], "value": o[1], "selected": default_value and o[1] == default_value} for o in static_options[0]] + elif static_options: + # Intersection based on values of multiple option connections. + intxn_vals = set.intersection(*({option[1] for option in options} for options in static_options)) + intxn_opts = {option for options in static_options for option in options if option[1] in intxn_vals} + d = defaultdict(set) # Collapse labels with same values + for label, value, _ in intxn_opts: + d[value].add(label) + options = [{"label": ', '.join(label), "value": value, "selected": default_value and value == default_value} for value, label in d.items()] + + return options + except Exception: + log.debug("Failed to generate options for text parameter, falling back to free text.", exc_info=True) + + def get_runtime_inputs(self, connections: Optional[Iterable[WorkflowStepConnection]] = None, **kwds): parameter_def = self._parse_state_into_dict() parameter_type = parameter_def["parameter_type"] optional = parameter_def["optional"] @@ -1028,38 +1059,12 @@ class InputParameterModule(WorkflowModule): # Really is just an attempt - tool module may not be available (small problem), get_options may really depend on other # values we are not setting, so this isn't great. Be sure to just fallback to text in this case. attemptRestrictOnConnections = is_text and parameter_def.get("restrictOnConnections") and connections - try: - if attemptRestrictOnConnections: - static_options = [] - # Retrieve possible runtime options for 'select' type inputs - for connection in connections: - # Well this isn't a great assumption... - module = connection.input_step.module - tool_inputs = module.tool.inputs # may not be set, but we're catching the Exception below. - - def callback(input, prefixed_name, context, **kwargs): - if prefixed_name == connection.input_name and hasattr(input, 'get_options'): - static_options.append(input.get_options(self.trans, {})) - visit_input_values(tool_inputs, module.state.inputs, callback) - - options = None - if static_options and len(static_options) == 1: - # If we are connected to a single option, just use it as is so order is preserved cleanly and such. - options = [{"label": o[0], "value": o[1], "selected": default_value and o[1] == default_value} for o in static_options[0]] - elif static_options: - # Intersection based on values of multiple option connections. - intxn_vals = set.intersection(*({option[1] for option in options} for options in static_options)) - intxn_opts = {option for options in static_options for option in options if option[1] in intxn_vals} - d = defaultdict(set) # Collapse labels with same values - for label, value, _ in intxn_opts: - d[value].add(label) - options = [{"label": ', '.join(label), "value": value, "selected": default_value and value == default_value} for value, label in d.items()] - - if options is not None: - parameter_kwds["options"] = options - restricted_inputs = True - except Exception: - log.debug("Failed to generate options for text parameter, falling back to free text.", exc_info=True) + if attemptRestrictOnConnections: + connections = cast(Iterable[WorkflowStepConnection], connections) + restricted_options = self.restrict_options(connections=connections, default_value=default_value) + if restricted_options is not None: + restricted_inputs = True + parameter_kwds["options"] = restricted_options def _parameter_def_list_to_options(parameter_value): options = [] From 8da2f9eb018a9ffb4a70c3a5ce3d93d76fc6392c Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 4 Feb 2022 15:16:32 +0100 Subject: [PATCH 004/652] Produce boolean --- lib/galaxy/workflow/modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 262c9bf961e..0bfa02e03dd 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -1028,7 +1028,7 @@ class InputParameterModule(WorkflowModule): options = None if static_options and len(static_options) == 1: # If we are connected to a single option, just use it as is so order is preserved cleanly and such. - options = [{"label": o[0], "value": o[1], "selected": default_value and o[1] == default_value} for o in static_options[0]] + options = [{"label": o[0], "value": o[1], "selected": bool(default_value and o[1] == default_value)} for o in static_options[0]] elif static_options: # Intersection based on values of multiple option connections. intxn_vals = set.intersection(*({option[1] for option in options} for options in static_options)) @@ -1036,7 +1036,7 @@ class InputParameterModule(WorkflowModule): d = defaultdict(set) # Collapse labels with same values for label, value, _ in intxn_opts: d[value].add(label) - options = [{"label": ', '.join(label), "value": value, "selected": default_value and value == default_value} for value, label in d.items()] + options = [{"label": ', '.join(label), "value": value, "selected": bool(default_value and value == default_value)} for value, label in d.items()] return options except Exception: From 43474de3682bd862a2774d8a642cf2be6f837876 Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Fri, 4 Feb 2022 15:33:32 +0100 Subject: [PATCH 005/652] Use Optional instead of Union with None MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: David López <46503462+davelopez@users.noreply.github.com> --- lib/galaxy/model/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 6af18041e32..ffbc61dedde 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6695,7 +6695,7 @@ class WorkflowStepConnection(Base, RepresentById): return self.input_step_input.name @property - def input_step(self) -> Union[WorkflowStep, None]: + def input_step(self) -> Optional[WorkflowStep]: return self.input_step_input and self.input_step_input.workflow_step @property From 2895dd36e1bb34c5072f9813143fd85d966cf50e Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 6 Feb 2022 10:41:35 +0100 Subject: [PATCH 006/652] Add config variable for setting tus upload directory --- doc/source/admin/galaxy_options.rst | 12 ++++++++++++ lib/galaxy/config/__init__.py | 5 +++++ lib/galaxy/config/sample/galaxy.yml.sample | 5 +++++ lib/galaxy/config/schemas/config_schema.yml | 8 ++++++++ lib/galaxy/tools/parameters/basic.py | 2 +- lib/galaxy/webapps/galaxy/buildapp.py | 2 +- 6 files changed, 32 insertions(+), 2 deletions(-) diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 93f7fafa547..4bcf2989ee2 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -2342,6 +2342,18 @@ :Type: str +~~~~~~~~~~~~~~~~~~~~ +``tus_upload_store`` +~~~~~~~~~~~~~~~~~~~~ + +:Description: + The upload store is a temporary directory in which files uploaded + by the tus middleware or server will be placed. Defaults to + new_file_path if not set. +:Default: ``None`` +:Type: str + + ~~~~~~~~~~~~~~~~~~~~~ ``chunk_upload_size`` ~~~~~~~~~~~~~~~~~~~~~ diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 0d76a4e39f6..49073dfd3fa 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -622,6 +622,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): involucro_path: str mulled_channels: List[str] nginx_upload_store: str + tus_upload_store: str pretty_datetime_format: str visualization_plugins_directory: str galaxy_infrastructure_url: str @@ -866,6 +867,9 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): if self.nginx_upload_store: self.nginx_upload_store = os.path.abspath(self.nginx_upload_store) + if self.tus_upload_store: + self.tus_upload_store = os.path.abspath(self.tus_upload_store) + self.object_store = kwargs.get('object_store', 'disk') self.object_store_check_old_style = string_as_bool(kwargs.get('object_store_check_old_style', False)) self.object_store_cache_path = self._in_root_dir(kwargs.get("object_store_cache_path", self._in_data_dir("object_store_cache"))) @@ -1095,6 +1099,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): self.managed_config_dir, self.new_file_path, self.nginx_upload_store, + self.tus_upload_store, self.object_store_cache_path, self.template_cache_path, self.tool_data_path, diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index e5b5e06decd..3017f8b4926 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -1189,6 +1189,11 @@ galaxy: # documentation for the corresponding nginx configuration. #nginx_upload_job_files_path: null + # The upload store is a temporary directory in which files uploaded by + # the tus middleware or server will be placed. Defaults to + # new_file_path if not set. + #tus_upload_store: null + # Galaxy can upload user files in chunks without using nginx. Enable # the chunk uploader by specifying a chunk size larger than 0. The # chunk size is specified in bytes (default: 10MB). diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index 7769e908e78..9a488e92337 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -1709,6 +1709,14 @@ mapping: operations on the remote end. See the Galaxy nginx documentation for the corresponding nginx configuration. + tus_upload_store: + type: str + required: False + desc: | + The upload store is a temporary directory in which files uploaded by the + tus middleware or server will be placed. + Defaults to new_file_path if not set. + chunk_upload_size: type: int default: 10485760 diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 5376554015f..2c31daee649 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -600,7 +600,7 @@ class FileToolParameter(ToolParameter): if 'session_id' in value: # handle api upload session_id = value["session_id"] - upload_store = trans.app.config.new_file_path + upload_store = trans.app.config.tus_upload_store or trans.app.config.new_file_path if re.match(r'^[\w-]+$', session_id) is None: raise ValueError("Invalid session id format.") local_filename = os.path.abspath(os.path.join(upload_store, session_id)) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index be5ba193ab6..7a1f3989a00 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -1391,7 +1391,7 @@ def wrap_in_middleware(app, global_conf, application_stack, **local_conf): # TUS upload middleware app = wrap_if_allowed(app, stack, TusMiddleware, kwargs={ 'upload_path': urljoin(f"{application_stack.config.galaxy_url_prefix}/", 'api/upload/resumable_upload'), - 'tmp_dir': application_stack.config.new_file_path, + 'tmp_dir': application_stack.config.tus_upload_store or application_stack.config.new_file_path, 'max_size': application_stack.config.maximum_upload_file_size }) # api batch call processing middleware From ee4603d9184f2dd8a97e372073bb341f2124f5dc Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 6 Feb 2022 19:16:44 +0100 Subject: [PATCH 007/652] Provide fallback for old chunked API --- lib/galaxy/tools/parameters/basic.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 2c31daee649..874fd2c2ef7 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -604,6 +604,9 @@ class FileToolParameter(ToolParameter): if re.match(r'^[\w-]+$', session_id) is None: raise ValueError("Invalid session id format.") local_filename = os.path.abspath(os.path.join(upload_store, session_id)) + if upload_store != trans.app.config.new_file_path and not os.path.exists(local_filename): + # Fallback for old chunked API, remove in 22.05 + local_filename = os.path.abspath(os.path.join(trans.app.config.new_file_path, session_id)) else: # handle nginx upload upload_store = trans.app.config.nginx_upload_store From 37846b2d279a102f9446baeac438d1c2212a8a03 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 7 Feb 2022 11:32:00 +0100 Subject: [PATCH 008/652] Make __link_file_check more robust There's no need to load the job and job parameters if the tool isn't upload1. This should fix https://github.com/galaxyproject/galaxy/issues/13311 --- lib/galaxy/jobs/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 82614ed7b42..bf0f8a42205 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -2258,10 +2258,10 @@ class JobWrapper(HasResourceParameters): method should be removed ASAP and replaced with some properly generic and stateful way of determining link-only datasets. -nate """ - if self.tool: + if self.tool and self.tool.id == 'upload1': job = self.get_job() param_dict = job.get_param_values(self.app) - return self.tool.id == 'upload1' and param_dict.get('link_data_only', None) == 'link_to_files' + return param_dict.get('link_data_only') == 'link_to_files' else: # The tool is unavailable, we try to move the outputs. return False From 83c0683e86733b9b4bdafd6184f50af2c82b0408 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 7 Feb 2022 11:12:54 +0100 Subject: [PATCH 009/652] Restore Metadata size limit Broke in https://github.com/galaxyproject/galaxy/pull/11902 --- lib/galaxy/model/mapping.py | 7 +++-- test/unit/data/test_metadata_limit.py | 44 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 test/unit/data/test_metadata_limit.py diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index c77fc982d4b..ceb2623774c 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -43,6 +43,7 @@ from galaxy import model from galaxy.model.base import SharedModelMapping from galaxy.model.custom_types import ( JSONType, + MetadataType, MutableJSONType, TrimmedString, UUIDType, @@ -248,7 +249,7 @@ model.HistoryDatasetAssociation.table = Table( Column("peek", TEXT, key="_peek"), Column("tool_version", TEXT), Column("extension", TrimmedString(64)), - Column("metadata", JSONType, key="_metadata"), + Column("metadata", MetadataType, key="_metadata"), Column("parent_id", Integer, ForeignKey("history_dataset_association.id"), nullable=True), Column("designation", TrimmedString(255)), Column("deleted", Boolean, index=True, default=False), @@ -271,7 +272,7 @@ model.HistoryDatasetAssociationHistory.table = Table( Column("version", Integer), Column("name", TrimmedString(255)), Column("extension", TrimmedString(64)), - Column("metadata", JSONType, key="_metadata"), + Column("metadata", MetadataType, key="_metadata"), Column("extended_metadata_id", Integer, ForeignKey("extended_metadata.id"), index=True), ) @@ -527,7 +528,7 @@ model.LibraryDatasetDatasetAssociation.table = Table( Column("peek", TEXT, key="_peek"), Column("tool_version", TEXT), Column("extension", TrimmedString(64)), - Column("metadata", JSONType, key="_metadata"), + Column("metadata", MetadataType, key="_metadata"), Column("parent_id", Integer, ForeignKey("library_dataset_dataset_association.id"), nullable=True), Column("designation", TrimmedString(255)), Column("deleted", Boolean, index=True, default=False), diff --git a/test/unit/data/test_metadata_limit.py b/test/unit/data/test_metadata_limit.py new file mode 100644 index 00000000000..e0c2121a2bc --- /dev/null +++ b/test/unit/data/test_metadata_limit.py @@ -0,0 +1,44 @@ +import pytest + +import galaxy.datatypes.registry as registry +import galaxy.model.mapping as mapping +from galaxy.model import ( + custom_types, + HistoryDatasetAssociation, + set_datatypes_registry, +) + +METADATA_LIMIT = 500 + + +@pytest.fixture(scope="module") +def datatypes_registry(): + r = registry.Registry() + r.load_datatypes() + set_datatypes_registry(r) + + +@pytest.fixture +def sa_session(datatypes_registry): + custom_types.MAX_METADATA_VALUE_SIZE = METADATA_LIMIT + return mapping.init("/tmp", "sqlite:///:memory:", create_tables=True).session + + +def create_bed_data(sa_session, string_size): + hda = HistoryDatasetAssociation(extension="bed") + big_string = "0" * string_size + sa_session.add(hda) + hda.metadata.column_names = [big_string] + assert hda.metadata.column_names + sa_session.flush() + return hda + + +def test_hda_below_limit(sa_session): + hda = create_bed_data(sa_session=sa_session, string_size=1) + assert len(hda.metadata.column_names[0]) == 1 + + +def test_hda_above_limit(sa_session): + hda = create_bed_data(sa_session=sa_session, string_size=1000) + assert not hda.metadata.column_names From 25d58f678dafd643aed29ad7d6e001b95ee131d8 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 7 Feb 2022 22:34:56 +0100 Subject: [PATCH 010/652] Drop unncessary log statement --- lib/galaxy/tools/parameters/dynamic_options.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/tools/parameters/dynamic_options.py b/lib/galaxy/tools/parameters/dynamic_options.py index b10af39e191..6eaac75fad4 100644 --- a/lib/galaxy/tools/parameters/dynamic_options.py +++ b/lib/galaxy/tools/parameters/dynamic_options.py @@ -157,7 +157,6 @@ class DataMetaFilter(Filter): self.column = d_option.column_spec_to_index(self.column) self.multiple = string_as_bool(elem.get("multiple", "False")) self.separator = elem.get("separator", ",") - log.error(f"data_meta.init: ref_name {self.ref_name} key {self.key} column {self.column} multiple {self.multiple} separator {self.separator}") def get_dependency_name(self): return self.ref_name From e6e4ae4030702cb70371d756bdea4b3ded87b0cb Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Mon, 7 Feb 2022 15:17:52 -0500 Subject: [PATCH 011/652] Add a check to see if the Notification API is available prior to trying to configure or send them. --- .../src/components/User/UserPreferences.vue | 24 +++++++++++-------- client/src/mvc/dataset/dataset-model.js | 22 ++++++++++------- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/client/src/components/User/UserPreferences.vue b/client/src/components/User/UserPreferences.vue index 6066a5bf348..c3a5e7b2a6a 100644 --- a/client/src/components/User/UserPreferences.vue +++ b/client/src/components/User/UserPreferences.vue @@ -177,16 +177,20 @@ export default { }, methods: { toggleNotifications() { - Notification.requestPermission().then(function (permission) { - //If the user accepts, let's create a notification - if (permission === "granted") { - new Notification("Notifications enabled", { - icon: "static/favicon.ico", - }); - } else { - alert("Notifications disabled, please re-enable through browser settings."); - } - }); + if (window.Notification) { + Notification.requestPermission().then(function (permission) { + //If the user accepts, let's create a notification + if (permission === "granted") { + new Notification("Notifications enabled", { + icon: "static/favicon.ico", + }); + } else { + alert("Notifications disabled, please re-enable through browser settings."); + } + }); + } else { + alert("Notifications are not supported by this browser."); + } }, openManageCustomBuilds() { const Galaxy = getGalaxyInstance(); diff --git a/client/src/mvc/dataset/dataset-model.js b/client/src/mvc/dataset/dataset-model.js index bc69f77b972..d968c3c7585 100644 --- a/client/src/mvc/dataset/dataset-model.js +++ b/client/src/mvc/dataset/dataset-model.js @@ -103,19 +103,25 @@ var DatasetAssociation = Backbone.Model.extend(BASE_MVC.LoggableMixin).extend( this.trigger("state:ready", currModel, newState, this.previous("state")); if (newState != "discarded") { if (newState === "ok") { - new Notification(`Job complete: ${this.get("name")}`, { - icon: "static/favicon.ico", - }); + // If Notifications are supported, send one. + if (window.Notification) { + new Notification(`Job complete: ${this.get("name")}`, { + icon: "static/favicon.ico", + }); + } if (TAB_UPDATES.is_hidden()) { TAB_UPDATES.hidden_count(hiddenupdates); hiddenupdates++; } } else if (newState == "error") { - new Notification(`Job failure: ${this.get("name")}`, { - icon: "static/erricon.ico", - }); - if (TAB_UPDATES.is_hidden() && Notification.permission == "granted") { - TAB_UPDATES.change_favicon("static/erricon.ico"); + // If Notifications are supported, send one. + if (window.Notification) { + new Notification(`Job failure: ${this.get("name")}`, { + icon: "static/erricon.ico", + }); + if (TAB_UPDATES.is_hidden() && Notification.permission == "granted") { + TAB_UPDATES.change_favicon("static/erricon.ico"); + } } } } From acd56abee8a993b44da06be66b728d147c97cefd Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 7 Feb 2022 22:35:19 +0100 Subject: [PATCH 012/652] Return DatasetChainModel Fixes failure to serialize the response. --- lib/galaxy/webapps/galaxy/api/datasets.py | 4 ++-- lib/galaxy/webapps/galaxy/services/datasets.py | 11 +++++++++-- lib/galaxy_test/api/test_history_contents.py | 15 +++++++++++++-- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/datasets.py b/lib/galaxy/webapps/galaxy/api/datasets.py index 2ef05428573..ce7a0a2efd1 100644 --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -47,7 +47,7 @@ from galaxy.webapps.galaxy.api.common import ( ) from galaxy.webapps.galaxy.services.datasets import ( ConvertedDatasetsMap, - DatasetInheritanceChainEntry, + DatasetInheritanceChain, DatasetsService, DatasetStorageDetails, DatasetTextContentDetails, @@ -122,7 +122,7 @@ class FastAPIDatasets: trans=DependsOnTrans, dataset_id: EncodedDatabaseIdField = DatasetIDPathParam, hda_ldda: DatasetSourceType = DatasetSourceQueryParam, - ) -> List[DatasetInheritanceChainEntry]: + ) -> DatasetInheritanceChain: return self.service.show_inheritance_chain(trans, dataset_id, hda_ldda) @router.get( diff --git a/lib/galaxy/webapps/galaxy/services/datasets.py b/lib/galaxy/webapps/galaxy/services/datasets.py index 7295078771f..39c155791e4 100644 --- a/lib/galaxy/webapps/galaxy/services/datasets.py +++ b/lib/galaxy/webapps/galaxy/services/datasets.py @@ -104,6 +104,13 @@ class DatasetInheritanceChainEntry(Model): ) +class DatasetInheritanceChain(Model): + __root__: List[DatasetInheritanceChainEntry] = Field( + default=[], + title="Dataset inheritance chain", + ) + + class ExtraFilesEntryClass(str, Enum): Directory = "Directory" File = "File" @@ -310,7 +317,7 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): trans: ProvidesHistoryContext, dataset_id: EncodedDatabaseIdField, hda_ldda: DatasetSourceType = DatasetSourceType.hda, - ) -> List[DatasetInheritanceChainEntry]: + ) -> DatasetInheritanceChain: """ Display inheritance chain for the given dataset. """ @@ -321,7 +328,7 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): for dep in inherit_chain: result.append(DatasetInheritanceChainEntry(name=f"{dep[0].name}", dep=dep[1])) - return result + return DatasetInheritanceChain(__root__=result) def update_permissions( self, diff --git a/lib/galaxy_test/api/test_history_contents.py b/lib/galaxy_test/api/test_history_contents.py index d1d83b7ab04..6cc73032a39 100644 --- a/lib/galaxy_test/api/test_history_contents.py +++ b/lib/galaxy_test/api/test_history_contents.py @@ -163,7 +163,7 @@ class HistoryContentsApiTestCase(ApiTestCase): self._assert_status_code_is(show_response, 200) self.__assert_matches_hda(hda1, show_response.json()) - def test_hda_copy(self): + def _create_copy(self): hda1 = self.dataset_populator.new_dataset(self.history_id) create_data = dict( source='hda', @@ -173,7 +173,18 @@ class HistoryContentsApiTestCase(ApiTestCase): assert self.__count_contents(second_history_id) == 0 create_response = self._post(f"histories/{second_history_id}/contents", create_data, json=True) self._assert_status_code_is(create_response, 200) - assert self.__count_contents(second_history_id) == 1 + return create_response.json() + + def test_hda_copy(self): + response = self._create_copy() + assert self.__count_contents(response['history_id']) == 1 + + def test_inheritance_chain(self): + response = self._create_copy() + inheritance_chain_response = self._get(f"datasets/{response['id']}/inheritance_chain") + self._assert_status_code_is_ok(inheritance_chain_response) + inheritance_chain = inheritance_chain_response.json() + assert len(inheritance_chain) == 1 def test_library_copy(self): ld = self.library_populator.new_library_dataset("lda_test_library") From bec9d4b49f4fec037b7f4ec46cea3dd36d15e12a Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 8 Feb 2022 12:36:28 +0100 Subject: [PATCH 013/652] Fix resuming job when job has optional data parameters This fixes ``` ERROR galaxy.tools.actions:__init__.py:683 Cannot remap rerun dependencies. Traceback (most recent call last): File "/Users/mvandenb/src/galaxy/lib/galaxy/tools/actions/__init__.py", line 664, in _remap_job_on_rerun self.__remap_parameters(job_to_remap, jtid, jtod, out_data) File "/Users/mvandenb/src/galaxy/lib/galaxy/tools/actions/__init__.py", line 694, in __remap_parameters input_values = {p.name: json.loads(p.value) for p in job_to_remap.parameters} File "/Users/mvandenb/src/galaxy/lib/galaxy/tools/actions/__init__.py", line 694, in input_values = {p.name: json.loads(p.value) for p in job_to_remap.parameters} File "/usr/local/Cellar/python@3.9/3.9.10/Frameworks/Python.framework/Versions/3.9/lib/python3.9/json/__init__.py", line 339, in loads raise TypeError(f'the JSON object must be str, bytes or bytearray, ' TypeError: the JSON object must be str, bytes or bytearray, not NoneType ``` Optional data inputs or optional selects are stored as `None` (super inconsistent, since most other parameters are stored as JOSN. We should create "basic_2.py" using pydantic at one point not too far into the future ...). This means we can't call `json.loads` on these. Fortunately this is the only place we do it, and we don't need to consider optional parameters here anyway. --- lib/galaxy/tools/actions/__init__.py | 5 +++-- lib/galaxy_test/api/test_workflows.py | 1 + test/functional/tools/identifier_multiple_in_conditional.xml | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index d622df5324d..9080b4fa798 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -675,12 +675,13 @@ class DefaultToolAction: return remapped_hdas def __remap_parameters(self, job_to_remap, jtid, jtod, out_data): - input_values = {p.name: json.loads(p.value) for p in job_to_remap.parameters} + input_values = {p.name: json.loads(p.value) for p in job_to_remap.parameters if p.value is not None} old_dataset_id = jtod.dataset_id new_dataset_id = out_data[jtod.name].id input_values = update_dataset_ids(input_values, {old_dataset_id: new_dataset_id}, src='hda') for p in job_to_remap.parameters: - p.value = json.dumps(input_values[p.name]) + if p.name in input_values: + p.value = json.dumps(input_values[p.name]) jtid.dataset = out_data[jtod.name] jtid.dataset.hid = jtod.dataset.hid log.info(f'Job {job_to_remap.id} input HDA {jtod.dataset.id} remapped to new HDA {jtid.dataset.id}') diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index ae0b636aecf..6f26a060ca4 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -967,6 +967,7 @@ steps: cond_param_inner: true input1: $link: 0/out_file1 + thedata: null cat: tool_id: cat1 in: diff --git a/test/functional/tools/identifier_multiple_in_conditional.xml b/test/functional/tools/identifier_multiple_in_conditional.xml index 81d12fd99fa..8bff2256f0b 100644 --- a/test/functional/tools/identifier_multiple_in_conditional.xml +++ b/test/functional/tools/identifier_multiple_in_conditional.xml @@ -16,6 +16,7 @@ + From 96e76c62d21208724efddfbdf5ff832646f716b0 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 7 Feb 2022 15:25:28 +0100 Subject: [PATCH 014/652] Don't record version command as part of command line Fixes https://github.com/galaxyproject/galaxy/issues/13306 --- lib/galaxy/jobs/__init__.py | 10 +++++--- lib/galaxy/metadata/set_metadata.py | 8 ++++++- lib/galaxy/tools/evaluation.py | 19 +++++++++------ lib/galaxy/tools/remote_tool_eval.py | 4 ++-- .../tools/version_command_plain.xml | 3 +++ test/unit/app/jobs/test_job_wrapper.py | 2 +- test/unit/app/tools/test_evaluation.py | 24 +++++++++---------- 7 files changed, 44 insertions(+), 26 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 7524e216adb..ba4740dbf56 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -939,6 +939,7 @@ class JobWrapper(HasResourceParameters): self.environment_variables: List[Dict[str, str]] = [] self.interactivetools: List[Dict[str, Any]] = [] self.command_line = None + self.version_command_line = None self._dependency_shell_commands = None # Tool versioning variables self.version_string = "" @@ -1192,7 +1193,7 @@ class JobWrapper(HasResourceParameters): tool_evaluator = self._get_tool_evaluator(job) compute_environment = compute_environment or self.default_compute_environment(job) tool_evaluator.set_compute_environment(compute_environment, get_special=get_special) - self.command_line, self.extra_filenames, self.environment_variables = tool_evaluator.build() + self.command_line, self.version_command_line, self.extra_filenames, self.environment_variables = tool_evaluator.build() job.command_line = self.command_line self.interactivetools = tool_evaluator.populate_interactivetools() self.app.interactivetool_manager.create_interactivetool(job, self.tool, self.interactivetools) @@ -1949,7 +1950,10 @@ class JobWrapper(HasResourceParameters): return has_output_limit or has_walltime_limit def get_command_line(self): - return self.command_line + """Return complete command line, including possible version command.""" + if self.remote_command_line: + return None + return f'{self.version_command_line or ""}{self.command_line}' def get_session_id(self): return self.session_id @@ -2305,7 +2309,7 @@ class TaskWrapper(JobWrapper): self.sa_session.flush() if not self.remote_command_line: - self.command_line, extra_filenames, self.environment_variables = tool_evaluator.build() + self.command_line, self.version_command_line, extra_filenames, self.environment_variables = tool_evaluator.build() self.extra_filenames.extend(extra_filenames) # Ensure galaxy_lib_dir is set in case there are any later chdirs diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py index 90755ce774c..6012fdc45c0 100644 --- a/lib/galaxy/metadata/set_metadata.py +++ b/lib/galaxy/metadata/set_metadata.py @@ -281,7 +281,13 @@ def set_metadata_portable(): job.state = final_job_state if os.path.exists(tool_script_file): with open(tool_script_file) as command_fh: - job.command_line = command_fh.read().strip() + command_line_lines = [] + for i, line in enumerate(command_fh): + if i == 0 and line.endswith('COMMAND_VERSION 2>&1;'): + # Don't record version command as part of command line + continue + command_line_lines.append(line) + job.command_line = "".join(command_line_lines).strip() export_store.export_job(job, include_job_data=False) unnamed_id_to_path = {} diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index 6f4eed1fa99..29d8c6b008a 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -91,6 +91,7 @@ class ToolEvaluator: self.param_dict: Dict[str, Any] = {} self.extra_filenames: List[str] = [] self.environment_variables: List[Dict[str, str]] = [] + self.version_command_line: Optional[str] = None self.command_line: Optional[str] = None def set_compute_environment(self, compute_environment: ComputeEnvironment, get_special: Optional[Callable] = None): @@ -445,8 +446,9 @@ class ToolEvaluator: global_tool_logs(self._build_config_files, config_file, "Building Config Files") global_tool_logs(self._build_param_file, config_file, 'Building Param File') global_tool_logs(self._build_command_line, config_file, "Building Command Line") + global_tool_logs(self._build_version_command, config_file, "Building Version Command Line") global_tool_logs(self._build_environment_variables, config_file, "Building Environment Variables") - return self.command_line, self.extra_filenames, self.environment_variables + return self.command_line, self.version_command_line, self.extra_filenames, self.environment_variables def _build_command_line(self): """ @@ -455,7 +457,6 @@ class ToolEvaluator: command = self.tool.command or '' param_dict = self.param_dict interpreter = self.tool.interpreter - version_string_cmd_raw = self.tool.version_string_cmd command_line = None if not command: return @@ -479,11 +480,14 @@ class ToolEvaluator: tool_dir = os.path.abspath(self.tool.tool_dir) abs_executable = os.path.join(tool_dir, executable) command_line = command_line.replace(executable, f"{interpreter} {shlex.quote(abs_executable)}", 1) + self.command_line = command_line + + def _build_version_command(self): + version_string_cmd_raw = self.tool.version_string_cmd if version_string_cmd_raw: version_command_template = string.Template(version_string_cmd_raw) - version_string_cmd = version_command_template.safe_substitute({"__tool_directory__": self.compute_environment.tool_directory()}) - command_line = f"{version_string_cmd} > {self.compute_environment.version_path()} 2>&1;\n{command_line}" - self.command_line = command_line + version_command = version_command_template.safe_substitute({"__tool_directory__": self.compute_environment.tool_directory()}) + self.version_command_line = f"{version_command} > {self.compute_environment.version_path()} 2>&1;\n" def _build_config_files(self): """ @@ -641,7 +645,7 @@ class PartialToolEvaluator(ToolEvaluator): def build(self): config_file = self.tool.config_file global_tool_logs(self._build_environment_variables, config_file, "Building Environment Variables") - return self.command_line, self.extra_filenames, self.environment_variables + return self.command_line, self.version_command_line, self.extra_filenames, self.environment_variables class RemoteToolEvaluator(ToolEvaluator): @@ -656,4 +660,5 @@ class RemoteToolEvaluator(ToolEvaluator): global_tool_logs(self._build_config_files, config_file, "Building Config Files") global_tool_logs(self._build_param_file, config_file, 'Building Param File') global_tool_logs(self._build_command_line, config_file, "Building Command Line") - return self.command_line, self.extra_filenames, self.environment_variables + global_tool_logs(self._build_version_command, config_file, "Building Version Command Line") + return self.command_line, self.version_command_line, self.extra_filenames, self.environment_variables diff --git a/lib/galaxy/tools/remote_tool_eval.py b/lib/galaxy/tools/remote_tool_eval.py index 8ffef968398..0235311023a 100644 --- a/lib/galaxy/tools/remote_tool_eval.py +++ b/lib/galaxy/tools/remote_tool_eval.py @@ -101,8 +101,8 @@ def main(TMPDIR, WORKING_DIRECTORY, IMPORT_STORE_DIRECTORY): tool_evaluator = evaluation.RemoteToolEvaluator(app=app, tool=tool, job=job_io.job, local_working_directory=WORKING_DIRECTORY) tool_evaluator.set_compute_environment(compute_environment=SharedComputeEnvironment(job_io=job_io, job=job_io.job)) with open(os.path.join(WORKING_DIRECTORY, 'tool_script.sh'), 'a') as out: - command_line, extra_filenames, environment_variables = tool_evaluator.build() - out.write(command_line) + command_line, version_command_line, extra_filenames, environment_variables = tool_evaluator.build() + out.write(f'{version_command_line or ""}{command_line}') if __name__ == "__main__": diff --git a/test/functional/tools/version_command_plain.xml b/test/functional/tools/version_command_plain.xml index b2ffc3e07ef..4d3b925624f 100644 --- a/test/functional/tools/version_command_plain.xml +++ b/test/functional/tools/version_command_plain.xml @@ -15,6 +15,9 @@ cp '$input' '$output' + + + diff --git a/test/unit/app/jobs/test_job_wrapper.py b/test/unit/app/jobs/test_job_wrapper.py index ba9cb55ca74..4566c8f8ba0 100644 --- a/test/unit/app/jobs/test_job_wrapper.py +++ b/test/unit/app/jobs/test_job_wrapper.py @@ -105,7 +105,7 @@ class MockEvaluator: pass def build(self): - return TEST_COMMAND, [], [] + return TEST_COMMAND, "", [], [] class MockJobQueue: diff --git a/test/unit/app/tools/test_evaluation.py b/test/unit/app/tools/test_evaluation.py index b6b0ce484c2..bd2e573dca5 100644 --- a/test/unit/app/tools/test_evaluation.py +++ b/test/unit/app/tools/test_evaluation.py @@ -53,7 +53,7 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): def test_simple_evaluation(self): self._setup_test_bwa_job() self._set_compute_environment() - command_line, extra_filenames, _ = self.evaluator.build() + command_line, version_command_line, extra_filenames, _ = self.evaluator.build() self.assertEqual(command_line, "bwa --thresh=4 --in=/galaxy/files/dataset_1.dat --out=/galaxy/files/dataset_2.dat") def test_repeat_evaluation(self): @@ -64,19 +64,19 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): self.job.parameters = [JobParameter(name="r", value='''[{"thresh": 4, "__index__": 0},{"thresh": 5, "__index__": 1}]''')] self.tool._command_line = "prog1 #for $r_i in $r # $r_i.thresh#end for#" self._set_compute_environment() - command_line, extra_filenames, _ = self.evaluator.build() + command_line, version_command_line, extra_filenames, _ = self.evaluator.build() self.assertEqual(command_line, "prog1 4 5") def test_eval_galaxy_url(self): self.tool._command_line = "prog1 $__galaxy_url__" self._set_compute_environment() - command_line, extra_filenames, _ = self.evaluator.build() + command_line, version_command_line, extra_filenames, _ = self.evaluator.build() self.assertEqual(command_line, "prog1 %s" % TEST_GALAXY_URL) def test_eval_history_id(self): self.tool._command_line = "prog1 '$__history_id__'" self._set_compute_environment() - command_line, extra_filenames, _ = self.evaluator.build() + command_line, version_command_line, extra_filenames, _ = self.evaluator.build() self.assertEqual(command_line, "prog1 '%s'" % self.app.security.encode_id(42)) def test_conditional_evaluation(self): @@ -94,7 +94,7 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): self.job.parameters = [JobParameter(name="c", value='''{"thresh": 4, "always_true": "true", "__current_case__": 0}''')] self.tool._command_line = "prog1 --thresh=${c.thresh} --test_param=${c.always_true}" self._set_compute_environment() - command_line, extra_filenames, _ = self.evaluator.build() + command_line, version_command_line, extra_filenames, _ = self.evaluator.build() self.assertEqual(command_line, "prog1 --thresh=4 --test_param=true") def test_evaluation_of_optional_datasets(self): @@ -106,7 +106,7 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): self.tool.set_params({"input1": parameter}) self.tool._command_line = "prog1 --opt_input='${input1}'" self._set_compute_environment() - command_line, extra_filenames, _ = self.evaluator.build() + command_line, version_command_line, extra_filenames, _ = self.evaluator.build() self.assertEqual(command_line, "prog1 --opt_input='None'") def test_evaluation_with_path_rewrites_wrapped(self): @@ -128,14 +128,14 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): input_paths=[DatasetPath(1, '/galaxy/files/dataset_1.dat', false_path=job_path_1)], output_paths=[DatasetPath(2, '/galaxy/files/dataset_2.dat', false_path=job_path_2)], ) - command_line, extra_filenames, _ = self.evaluator.build() + command_line, version_command_line, extra_filenames, _ = self.evaluator.build() self.assertEqual(command_line, f"bwa --thresh=4 --in={job_path_1} --out={job_path_2}") def test_configfiles_evaluation(self): self.tool.config_files.append(("conf1", None, "$thresh")) self.tool._command_line = "prog1 $conf1" self._set_compute_environment() - command_line, extra_filenames, _ = self.evaluator.build() + command_line, version_command_line, extra_filenames, _ = self.evaluator.build() self.assertEqual(len(extra_filenames), 1) config_filename = extra_filenames[0] config_basename = os.path.basename(config_filename) @@ -175,15 +175,15 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): }) self.tool._command_line = "prog1 $index_path.fields.path" self._set_compute_environment(unstructured_path_rewrites={"/old": "/new"}) - command_line, extra_filenames, _ = self.evaluator.build() + command_line, version_command_line, extra_filenames, _ = self.evaluator.build() self.assertEqual(command_line, "prog1 /new/path/human") def test_version_command(self): self.tool.version_string_cmd = "echo v.1.1" self._setup_test_bwa_job() self._set_compute_environment() - command_line, extra_filenames, _ = self.evaluator.build() - assert self.tool.version_string_cmd in command_line + command_line, version_command_line, extra_filenames, _ = self.evaluator.build() + assert self.tool.version_string_cmd in version_command_line def test_template_property_app(self): self._assert_template_property_is("$__app__.config.new_file_path", self.app.config.new_file_path) @@ -201,7 +201,7 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): self.tool._command_line = "test.exe" self.tool.config_files.append(("conf1", None, """%s""" % expression)) self._set_compute_environment() - _, extra_filenames, _ = self.evaluator.build() + _, _, extra_filenames, _ = self.evaluator.build() config_filename = extra_filenames[0] self.assertEqual(open(config_filename).read(), value) From 9cb0c45ca4c1dea8667ee5ddd6dc3a2460fc0b8e Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 8 Feb 2022 09:57:50 -0500 Subject: [PATCH 015/652] Add .git-blame-ignore-revs to 21.09. Without this, if you have set the file up in git config blame will blow up with something like `could not open object name list: .git-blame-ignore-revs`. I only included the relevant hashes here, will need to handle a merge as it goes forward. --- .git-blame-ignore-revs | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .git-blame-ignore-revs diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000000..29a799793ca --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,3 @@ +# Migrate code style to Prettier +5b2928f851bd5ea3b9c2a04abf2cee9ff0bc54cc +87873c5e2f4e6b97fe0f2084bfca0295fcd471de From 7c4905251cb647bf3eca836c5f207389b5ed20cd Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Tue, 8 Feb 2022 21:35:13 +0100 Subject: [PATCH 016/652] Get the needed ToolEvaluator.build return value by index Co-authored-by: Nicola Soranzo --- test/unit/app/tools/test_evaluation.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/unit/app/tools/test_evaluation.py b/test/unit/app/tools/test_evaluation.py index bd2e573dca5..6f2a709cc32 100644 --- a/test/unit/app/tools/test_evaluation.py +++ b/test/unit/app/tools/test_evaluation.py @@ -53,7 +53,7 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): def test_simple_evaluation(self): self._setup_test_bwa_job() self._set_compute_environment() - command_line, version_command_line, extra_filenames, _ = self.evaluator.build() + command_line = self.evaluator.build()[0] self.assertEqual(command_line, "bwa --thresh=4 --in=/galaxy/files/dataset_1.dat --out=/galaxy/files/dataset_2.dat") def test_repeat_evaluation(self): @@ -64,19 +64,19 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): self.job.parameters = [JobParameter(name="r", value='''[{"thresh": 4, "__index__": 0},{"thresh": 5, "__index__": 1}]''')] self.tool._command_line = "prog1 #for $r_i in $r # $r_i.thresh#end for#" self._set_compute_environment() - command_line, version_command_line, extra_filenames, _ = self.evaluator.build() + command_line = self.evaluator.build()[0] self.assertEqual(command_line, "prog1 4 5") def test_eval_galaxy_url(self): self.tool._command_line = "prog1 $__galaxy_url__" self._set_compute_environment() - command_line, version_command_line, extra_filenames, _ = self.evaluator.build() + command_line = self.evaluator.build()[0] self.assertEqual(command_line, "prog1 %s" % TEST_GALAXY_URL) def test_eval_history_id(self): self.tool._command_line = "prog1 '$__history_id__'" self._set_compute_environment() - command_line, version_command_line, extra_filenames, _ = self.evaluator.build() + command_line = self.evaluator.build()[0] self.assertEqual(command_line, "prog1 '%s'" % self.app.security.encode_id(42)) def test_conditional_evaluation(self): @@ -94,7 +94,7 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): self.job.parameters = [JobParameter(name="c", value='''{"thresh": 4, "always_true": "true", "__current_case__": 0}''')] self.tool._command_line = "prog1 --thresh=${c.thresh} --test_param=${c.always_true}" self._set_compute_environment() - command_line, version_command_line, extra_filenames, _ = self.evaluator.build() + command_line = self.evaluator.build()[0] self.assertEqual(command_line, "prog1 --thresh=4 --test_param=true") def test_evaluation_of_optional_datasets(self): @@ -106,7 +106,7 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): self.tool.set_params({"input1": parameter}) self.tool._command_line = "prog1 --opt_input='${input1}'" self._set_compute_environment() - command_line, version_command_line, extra_filenames, _ = self.evaluator.build() + command_line = self.evaluator.build()[0] self.assertEqual(command_line, "prog1 --opt_input='None'") def test_evaluation_with_path_rewrites_wrapped(self): @@ -128,14 +128,14 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): input_paths=[DatasetPath(1, '/galaxy/files/dataset_1.dat', false_path=job_path_1)], output_paths=[DatasetPath(2, '/galaxy/files/dataset_2.dat', false_path=job_path_2)], ) - command_line, version_command_line, extra_filenames, _ = self.evaluator.build() + command_line = self.evaluator.build()[0] self.assertEqual(command_line, f"bwa --thresh=4 --in={job_path_1} --out={job_path_2}") def test_configfiles_evaluation(self): self.tool.config_files.append(("conf1", None, "$thresh")) self.tool._command_line = "prog1 $conf1" self._set_compute_environment() - command_line, version_command_line, extra_filenames, _ = self.evaluator.build() + command_line, _, extra_filenames, _ = self.evaluator.build() self.assertEqual(len(extra_filenames), 1) config_filename = extra_filenames[0] config_basename = os.path.basename(config_filename) @@ -175,14 +175,14 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): }) self.tool._command_line = "prog1 $index_path.fields.path" self._set_compute_environment(unstructured_path_rewrites={"/old": "/new"}) - command_line, version_command_line, extra_filenames, _ = self.evaluator.build() + command_line = self.evaluator.build()[0] self.assertEqual(command_line, "prog1 /new/path/human") def test_version_command(self): self.tool.version_string_cmd = "echo v.1.1" self._setup_test_bwa_job() self._set_compute_environment() - command_line, version_command_line, extra_filenames, _ = self.evaluator.build() + version_command_line = self.evaluator.build()[1] assert self.tool.version_string_cmd in version_command_line def test_template_property_app(self): @@ -201,7 +201,7 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): self.tool._command_line = "test.exe" self.tool.config_files.append(("conf1", None, """%s""" % expression)) self._set_compute_environment() - _, _, extra_filenames, _ = self.evaluator.build() + extra_filenames = self.evaluator.build()[2] config_filename = extra_filenames[0] self.assertEqual(open(config_filename).read(), value) From 991f82b136ec247745feb591701ea8b50b19e925 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 8 Feb 2022 22:56:17 +0100 Subject: [PATCH 017/652] Fix history exports import by url --- lib/galaxy/webapps/galaxy/api/histories.py | 4 ++-- lib/galaxy_test/base/populators.py | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index 2628d6a5f95..4f58ef4993c 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -59,7 +59,7 @@ from galaxy.web import ( expose_api, expose_api_anonymous, expose_api_anonymous_and_sessionless, - expose_api_raw, + expose_api_raw_anonymous, ) from galaxy.webapps.galaxy.api.common import ( parse_serialization_params, @@ -765,7 +765,7 @@ class HistoriesController(BaseGalaxyAPIController): trans.response.status = 202 return export_result - @expose_api_raw + @expose_api_raw_anonymous def archive_download(self, trans, id, jeha_id, **kwds): """ GET /api/histories/{id}/exports/{jeha_id} diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index 78d70832ee7..fa3417054c7 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -913,6 +913,11 @@ class BaseDatasetPopulator(BasePopulator): assert update_response.status_code == 200, update_response.content return update_response.json() + def make_public(self, history_id: str) -> dict: + sharing_response = self._put(f"histories/{history_id}/publish") + assert sharing_response.status_code == 200 + return sharing_response.json() + def validate_dataset(self, history_id: str, dataset_id: str) -> Dict[str, Any]: url = f"histories/{history_id}/contents/{dataset_id}/validate" update_response = self._put(url) @@ -1035,13 +1040,12 @@ class BaseDatasetPopulator(BasePopulator): return len(contents) def reimport_history(self, history_id, history_name, wait_on_history_length, export_kwds, api_key): + # Make history public so we can import by url + self.make_public(history_id) # Export the history. - download_path = self.export_url(history_id, export_kwds, api_key, check_download=True) + download_url = self.export_url(history_id, export_kwds, api_key, check_download=True) - # Create download for history - full_download_url = urllib.parse.urljoin(download_path, f"?key={api_key}") - - import_data = dict(archive_source=full_download_url, archive_type="url") + import_data = dict(archive_source=download_url, archive_type="url") imported_history_id = self.import_history_and_wait_for_name(import_data, history_name) From 1659d0943c6deb64c90b29a5f6bacb537b8e5b0d Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Wed, 9 Feb 2022 11:55:28 +0100 Subject: [PATCH 018/652] block workflow invocation if wrong tool versions installed --- .../Workflow/Run/WorkflowRunForm.vue | 3 ++ .../Workflow/Run/WorkflowRunFormSimple.vue | 1 + lib/galaxy/managers/workflows.py | 11 ++++---- lib/galaxy/webapps/galaxy/api/workflows.py | 28 ++++++++++++++----- lib/galaxy_test/api/test_workflows.py | 25 +++++++++++++++-- 5 files changed, 53 insertions(+), 15 deletions(-) diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue index 8b63094e327..a65441c99dd 100644 --- a/client/src/components/Workflow/Run/WorkflowRunForm.vue +++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue @@ -206,6 +206,9 @@ export default { // Tool form always wants a list of invocations back // so that inputs can be batched. batch: true, + // the user is already warned if tool versions are wrong, + // they can still choose to invoke the workflow anyway. + require_exact_tool_versions: false, }; console.debug("WorkflowRunForm::onExecute()", "Ready for submission.", jobDef); diff --git a/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue b/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue index e95f828c33a..6d8543f0e66 100644 --- a/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue +++ b/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue @@ -114,6 +114,7 @@ export default { inputs_by: "step_index", batch: true, use_cached_job: this.useJobCache, + require_exact_tool_versions: false, }; if (this.targetHistory == "current") { data.history_id = this.model.historyId; diff --git a/lib/galaxy/managers/workflows.py b/lib/galaxy/managers/workflows.py index 19fb78ce4bf..2e827d23256 100644 --- a/lib/galaxy/managers/workflows.py +++ b/lib/galaxy/managers/workflows.py @@ -1581,15 +1581,16 @@ class WorkflowContentsManager(UsesAnnotations): dry_run=refactor_request.dry_run, ) - def get_all_tool_ids(self, workflow): - tool_ids = set() + def get_all_tools(self, workflow): + tools = [] for step in workflow.steps: if step.type == "tool": if step.tool_id: - tool_ids.add(step.tool_id) + if {"tool_id": step.tool_id, "tool_version": step.tool_version} not in tools: + tools.append({"tool_id": step.tool_id, "tool_version": step.tool_version}) elif step.type == "subworkflow": - tool_ids.update(self.get_all_tool_ids(step.subworkflow)) - return tool_ids + tools.extend(self.get_all_tools(step.subworkflow)) + return tools class RefactorRequest(RefactorActions): diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 7382f476577..0cea440c1a8 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -215,10 +215,12 @@ class WorkflowsAPIController(BaseGalaxyAPIController, UsesStoredWorkflowMixin, U workflows = [] workflows_by_toolshed = dict() for value in rval: - tool_ids = self.workflow_contents_manager.get_all_tool_ids( + tools = self.workflow_contents_manager.get_all_tools( self.__get_stored_workflow(trans, value["id"]).latest_workflow ) - missing_tool_ids = [tool_id for tool_id in tool_ids if self.app.toolbox.is_missing_shed_tool(tool_id)] + missing_tool_ids = [ + tool["tool_id"] for tool in tools if self.app.toolbox.is_missing_shed_tool(tool["tool_id"]) + ] if len(missing_tool_ids) > 0: value["missing_tools"] = missing_tool_ids workflows_missing_tools.append(value) @@ -808,12 +810,24 @@ class WorkflowsAPIController(BaseGalaxyAPIController, UsesStoredWorkflowMixin, U if not is_batch and len(run_configs) != 1: raise exceptions.RequestParameterInvalidException("Must specify 'batch' to use batch parameters.") - tool_ids = self.workflow_contents_manager.get_all_tool_ids(workflow) - missing_tool_ids = [tool_id for tool_id in tool_ids if not self.app.toolbox.has_tool(tool_id)] - if missing_tool_ids: - raise exceptions.MessageException( - f"Workflow was not invoked; the following required tools are not installed: {', '.join(missing_tool_ids)}" + require_exact_tool_versions = util.string_as_bool(payload.get("require_exact_tool_versions", "true")) + tools = self.workflow_contents_manager.get_all_tools(workflow) + missing_tools = [ + tool + for tool in tools + if not self.app.toolbox.has_tool( + tool["tool_id"], tool_version=tool["tool_version"], exact=require_exact_tool_versions ) + ] + if missing_tools: + missing_tools_message = "Workflow was not invoked; the following required tools are not installed: " + if require_exact_tool_versions: + missing_tools_message += ", ".join( + [f"{tool['tool_id']} (version {tool['tool_version']})" for tool in missing_tools] + ) + else: + missing_tools_message += ", ".join([tool["tool_id"] for tool in missing_tools]) + raise exceptions.MessageException(missing_tools_message) invocations = [] for run_config in run_configs: diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index cc8102cce6f..f573de9d13a 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -31,6 +31,7 @@ from galaxy_test.base.populators import ( wait_on, WorkflowPopulator, ) +from galaxy_test.base.uses_shed import UsesShed from galaxy_test.base.workflow_fixtures import ( WORKFLOW_NESTED_REPLACEMENT_PARAMETER, WORKFLOW_NESTED_RUNTIME_PARAMETER, @@ -228,7 +229,7 @@ input1: # - Allow post to workflows//run in addition to posting to # /workflows with id in payload. # - Much more testing obviously, always more testing. -class WorkflowsApiTestCase(BaseWorkflowsApiTestCase, ChangeDatatypeTestCase): +class WorkflowsApiTestCase(BaseWorkflowsApiTestCase, ChangeDatatypeTestCase, UsesShed): def test_show_valid(self): workflow_id = self.workflow_populator.simple_workflow("dummy") workflow_id = self.workflow_populator.simple_workflow("test_regular") @@ -920,17 +921,35 @@ steps: assert invocation["state"] == "scheduled", invocation def test_run_workflow_with_missing_tool(self): + self.install_repository("iuc", "compose_text_param", "feb3acba1e0a") # 0.1.0 with self.dataset_populator.test_history() as history_id: workflow_id = self._upload_yaml_workflow( """ class: GalaxyWorkflow steps: - step1: + nonexistent: tool_id: nonexistent_tool tool_version: "0.1" + label: nonexistent + compose_text_param: + tool_id: compose_text_param + tool_version: 0.0.1 + label: compose_text_param """ ) - invocation_response = self.__invoke_workflow(workflow_id, history_id=history_id, assert_ok=False) + # should fail and return both tool ids since version 0.0.1 of compose_text_param does not exist + invocation_response = self.__invoke_workflow( + workflow_id, history_id=history_id, assert_ok=False, request={"require_exact_tool_versions": True} + ) + self._assert_status_code_is(invocation_response, 400) + self.assertEqual( + invocation_response.json().get("err_msg"), + "Workflow was not invoked; the following required tools are not installed: nonexistent_tool (version 0.1), compose_text_param (version 0.0.1)", + ) + # should fail but return only the tool_id of non_existent tool as another version of compose_text_param is installed + invocation_response = self.__invoke_workflow( + workflow_id, history_id=history_id, assert_ok=False, request={"require_exact_tool_versions": False} + ) self._assert_status_code_is(invocation_response, 400) self.assertEqual( invocation_response.json().get("err_msg"), From 6de7d6f392cc8b2f7fc6332c5204da1fbde07f66 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Wed, 9 Feb 2022 15:15:39 +0100 Subject: [PATCH 019/652] fix small bug in toolbox's get_tool() so non-shed tool can be used for missing tool workflow invocation test --- lib/galaxy/tool_util/toolbox/base.py | 2 +- lib/galaxy_test/api/test_workflows.py | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py index 775c31210ee..8b0b803cd73 100644 --- a/lib/galaxy/tool_util/toolbox/base.py +++ b/lib/galaxy/tool_util/toolbox/base.py @@ -668,7 +668,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): return self._tools_by_id[tool_id] elif tool_version in self._tool_versions_by_id[tool_id]: return self._tool_versions_by_id[tool_id][tool_version] - elif exact: + if exact: # We're looking for an exact match, so we skip lineage and # versionless mapping, though we may want to check duplicate # toolsheds diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index f573de9d13a..53ba9058ac1 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -31,7 +31,6 @@ from galaxy_test.base.populators import ( wait_on, WorkflowPopulator, ) -from galaxy_test.base.uses_shed import UsesShed from galaxy_test.base.workflow_fixtures import ( WORKFLOW_NESTED_REPLACEMENT_PARAMETER, WORKFLOW_NESTED_RUNTIME_PARAMETER, @@ -229,7 +228,7 @@ input1: # - Allow post to workflows//run in addition to posting to # /workflows with id in payload. # - Much more testing obviously, always more testing. -class WorkflowsApiTestCase(BaseWorkflowsApiTestCase, ChangeDatatypeTestCase, UsesShed): +class WorkflowsApiTestCase(BaseWorkflowsApiTestCase, ChangeDatatypeTestCase): def test_show_valid(self): workflow_id = self.workflow_populator.simple_workflow("dummy") workflow_id = self.workflow_populator.simple_workflow("test_regular") @@ -921,7 +920,6 @@ steps: assert invocation["state"] == "scheduled", invocation def test_run_workflow_with_missing_tool(self): - self.install_repository("iuc", "compose_text_param", "feb3acba1e0a") # 0.1.0 with self.dataset_populator.test_history() as history_id: workflow_id = self._upload_yaml_workflow( """ @@ -932,21 +930,21 @@ steps: tool_version: "0.1" label: nonexistent compose_text_param: - tool_id: compose_text_param - tool_version: 0.0.1 - label: compose_text_param + tool_id: multiple_versions + tool_version: 0.3 + label: multiple_versions """ ) - # should fail and return both tool ids since version 0.0.1 of compose_text_param does not exist + # should fail and return both tool ids since version 0.3 of multiple_versions does not exist invocation_response = self.__invoke_workflow( workflow_id, history_id=history_id, assert_ok=False, request={"require_exact_tool_versions": True} ) self._assert_status_code_is(invocation_response, 400) self.assertEqual( invocation_response.json().get("err_msg"), - "Workflow was not invoked; the following required tools are not installed: nonexistent_tool (version 0.1), compose_text_param (version 0.0.1)", + "Workflow was not invoked; the following required tools are not installed: nonexistent_tool (version 0.1), multiple_versions (version 0.3)", ) - # should fail but return only the tool_id of non_existent tool as another version of compose_text_param is installed + # should fail but return only the tool_id of non_existent tool as another version of multiple_versions is installed invocation_response = self.__invoke_workflow( workflow_id, history_id=history_id, assert_ok=False, request={"require_exact_tool_versions": False} ) From 925f44a16ea48408e4a61cc89f2795dce415cfd8 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 9 Feb 2022 17:30:12 +0100 Subject: [PATCH 020/652] Improve (history import) job error reporting --- lib/galaxy_test/base/populators.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index fa3417054c7..5e42dbbe301 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -555,7 +555,7 @@ class BaseDatasetPopulator(BasePopulator): self.wait_for_history(history_id, assert_ok=True, timeout=timeout) def wait_for_job(self, job_id: str, assert_ok: bool = False, timeout: timeout_type = DEFAULT_TIMEOUT): - return wait_on_state(lambda: self.get_job_details(job_id), desc="job state", assert_ok=assert_ok, timeout=timeout) + return wait_on_state(lambda: self.get_job_details(job_id, full=True), desc="job state", assert_ok=assert_ok, timeout=timeout) def get_job_details(self, job_id: str, full: bool = False) -> Response: return self._get(f"jobs/{job_id}?full={full}") @@ -996,6 +996,7 @@ class BaseDatasetPopulator(BasePopulator): files["archive_file"] = archive_file import_response = self._post("histories", data=import_data, files=files) api_asserts.assert_status_code_is(import_response, 200) + return import_response.json()['id'] def import_history_and_wait_for_name(self, import_data, history_name): def history_names(): @@ -1004,7 +1005,8 @@ class BaseDatasetPopulator(BasePopulator): import_name = f"imported from archive: {history_name}" assert import_name not in history_names() - self.import_history(import_data) + job_id = self.import_history(import_data) + self.wait_for_job(job_id, assert_ok=True) def has_history_with_name(): histories = history_names() @@ -2155,12 +2157,13 @@ def wait_on_state(state_func: Callable, desc="state", skip_states=None, ok_state def get_state(): response = state_func() assert response.status_code == 200, f"Failed to fetch state update while waiting. [{response.content}]" - state = response.json()["state"] + state_response = response.json() + state = state_response["state"] if state in skip_states: return None else: if assert_ok: - assert state in ok_states, f"Final state - {state} - not okay." + assert state in ok_states, f"Final state - {state} - not okay. Full response: {state_response}" return state if skip_states is None: From db6e2db4dc99ed4bdce529c71200b19df1ad6911 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 9 Feb 2022 17:30:34 +0100 Subject: [PATCH 021/652] Endpoint needs to be sessionless as well --- lib/galaxy/webapps/galaxy/api/histories.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index 4f58ef4993c..d17bee888a0 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -59,7 +59,7 @@ from galaxy.web import ( expose_api, expose_api_anonymous, expose_api_anonymous_and_sessionless, - expose_api_raw_anonymous, + expose_api_raw_anonymous_and_sessionless, ) from galaxy.webapps.galaxy.api.common import ( parse_serialization_params, @@ -765,7 +765,7 @@ class HistoriesController(BaseGalaxyAPIController): trans.response.status = 202 return export_result - @expose_api_raw_anonymous + @expose_api_raw_anonymous_and_sessionless def archive_download(self, trans, id, jeha_id, **kwds): """ GET /api/histories/{id}/exports/{jeha_id} From 96d89a4f5c01e688e89621cb4c7504ed5dc9a8f7 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 9 Feb 2022 17:38:10 +0100 Subject: [PATCH 022/652] Drop API key from more places --- lib/galaxy_test/api/test_histories.py | 2 +- lib/galaxy_test/api/test_workflow_extraction.py | 2 +- lib/galaxy_test/base/populators.py | 13 ++++++------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/galaxy_test/api/test_histories.py b/lib/galaxy_test/api/test_histories.py index fc7983d10a9..4d1f08c842b 100644 --- a/lib/galaxy_test/api/test_histories.py +++ b/lib/galaxy_test/api/test_histories.py @@ -390,7 +390,7 @@ class ImportExportTests(BaseHistories): self.dataset_populator.wait_for_history(history_id, assert_ok=assert_ok) return self.dataset_populator.reimport_history( - history_id, history_name, wait_on_history_length=wait_on_history_length, export_kwds=export_kwds, api_key=self.galaxy_interactor.api_key + history_id, history_name, wait_on_history_length=wait_on_history_length, export_kwds=export_kwds, ) def _import_history_and_wait(self, import_data, history_name, wait_on_history_length=None): diff --git a/lib/galaxy_test/api/test_workflow_extraction.py b/lib/galaxy_test/api/test_workflow_extraction.py index 1a4a34ae504..7d509c0d9e5 100644 --- a/lib/galaxy_test/api/test_workflow_extraction.py +++ b/lib/galaxy_test/api/test_workflow_extraction.py @@ -499,7 +499,7 @@ test_data: history_length = self.dataset_populator.history_length(history_id) new_history_id = self.dataset_populator.reimport_history( - history_id, history_name, wait_on_history_length=history_length, export_kwds={}, api_key=self.galaxy_interactor.api_key + history_id, history_name, wait_on_history_length=history_length, export_kwds={}, ) # wait a little more for those jobs, todo fix to wait for history imported false or # for a specific number of jobs... diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index 5e42dbbe301..9183483c2c5 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -972,20 +972,19 @@ class BaseDatasetPopulator(BasePopulator): assert "job_id" in job_desc return self.wait_for_job(job_desc["job_id"]) - def export_url(self, history_id: str, data, api_key: str, check_download: bool = True) -> str: + def export_url(self, history_id: str, data, check_download: bool = True) -> str: put_response = self.prepare_export(history_id, data) response = put_response.json() api_asserts.assert_has_keys(response, "download_url") download_url = urllib.parse.urljoin(self.galaxy_interactor.api_url, response["download_url"].strip('/')) if check_download: - self.get_export_url(download_url, api_key) + self.get_export_url(download_url) return download_url - def get_export_url(self, export_url, api_key) -> Response: - full_download_url = f"{export_url}?key={api_key}" - download_response = self._get(full_download_url) + def get_export_url(self, export_url) -> Response: + download_response = self._get(export_url) api_asserts.assert_status_code_is(download_response, 200) return download_response @@ -1041,11 +1040,11 @@ class BaseDatasetPopulator(BasePopulator): contents = contents_response.json() return len(contents) - def reimport_history(self, history_id, history_name, wait_on_history_length, export_kwds, api_key): + def reimport_history(self, history_id, history_name, wait_on_history_length, export_kwds): # Make history public so we can import by url self.make_public(history_id) # Export the history. - download_url = self.export_url(history_id, export_kwds, api_key, check_download=True) + download_url = self.export_url(history_id, export_kwds, check_download=True) import_data = dict(archive_source=download_url, archive_type="url") From 87337bad17c2cc7698a3ad150def0a1cf4ebc273 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 9 Feb 2022 19:16:13 +0100 Subject: [PATCH 023/652] allow univa destinations wo nativespec if no nativespec is given for a univa destination the runner does not know how much run time / memory was given to a job which led to a comparison with `None`. Now we default to infinite runtime and memory. --- lib/galaxy/jobs/runners/univa.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/jobs/runners/univa.py b/lib/galaxy/jobs/runners/univa.py index 645a325dc2e..21999e831fa 100644 --- a/lib/galaxy/jobs/runners/univa.py +++ b/lib/galaxy/jobs/runners/univa.py @@ -32,6 +32,7 @@ import logging import re import signal import time +from math import inf from galaxy.jobs.runners.drmaa import DRMAAJobRunner from galaxy.util import ( @@ -566,8 +567,8 @@ def _parse_native_specs(job_id, native_spec): specification string passed to GE return time,mem (or None,None if nothing found) """ - tme = None - mem = None + tme = inf + mem = inf # parse time m = re.search(r"rt=([0-9:]+)[\s,]*", native_spec) if m is not None: From 6ba67b159f8563a5d578d28c25f9664fc64803c3 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Thu, 10 Feb 2022 11:28:17 +0100 Subject: [PATCH 024/652] revert change to toolbox's get_tool() --- lib/galaxy/tool_util/toolbox/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py index 8b0b803cd73..ebc801fd435 100644 --- a/lib/galaxy/tool_util/toolbox/base.py +++ b/lib/galaxy/tool_util/toolbox/base.py @@ -668,7 +668,9 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): return self._tools_by_id[tool_id] elif tool_version in self._tool_versions_by_id[tool_id]: return self._tool_versions_by_id[tool_id][tool_version] - if exact: + # should be if exact=True not elif? Otherwise we can end up doing non-exact searches even + # if exact=True. Anyway, changing it breaks a lot of tests involving built-in tools + elif exact: # We're looking for an exact match, so we skip lineage and # versionless mapping, though we may want to check duplicate # toolsheds From 865a3b915e84cb2ccc62e28647450fdd79a0c121 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Thu, 10 Feb 2022 11:30:36 +0100 Subject: [PATCH 025/652] add integration test for invoking workflows with nonexistent tool_ids or tool_versions --- test/integration/test_workflow_invocation.py | 55 ++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 test/integration/test_workflow_invocation.py diff --git a/test/integration/test_workflow_invocation.py b/test/integration/test_workflow_invocation.py new file mode 100644 index 00000000000..f8d877207e6 --- /dev/null +++ b/test/integration/test_workflow_invocation.py @@ -0,0 +1,55 @@ +"""Integration tests for workflow syncing.""" + +from galaxy_test.base.populators import ( + DatasetPopulator, + WorkflowPopulator, +) +from galaxy_test.base.uses_shed import UsesShed +from galaxy_test.driver import integration_util + + +class WorkflowInvocationTestCase(integration_util.IntegrationTestCase, UsesShed): + + framework_tool_and_types = True + require_admin_user = False + + def setUp(self): + super().setUp() + self.dataset_populator = DatasetPopulator(self.galaxy_interactor) + self.workflow_populator = WorkflowPopulator(self.galaxy_interactor) + + def test_run_workflow_with_missing_tool(self): + self.install_repository("iuc", "compose_text_param", "feb3acba1e0a") # 0.1.0 + with self.dataset_populator.test_history() as history_id: + workflow_id = self.workflow_populator.upload_yaml_workflow( + """ +class: GalaxyWorkflow +steps: + nonexistent: + tool_id: nonexistent_tool + tool_version: "0.1" + label: nonexistent + compose_text_param: + tool_id: compose_text_param + tool_version: "0.0.1" + label: compose_text_param +""" + ) + # should fail and return both tool ids since version 0.0.1 of compose_text_param does not exist + invocation_response = self.workflow_populator.invoke_workflow( + workflow_id, history_id=history_id, assert_ok=False, request={"require_exact_tool_versions": True} + ) + self._assert_status_code_is(invocation_response, 400) + self.assertEqual( + invocation_response.json().get("err_msg"), + "Workflow was not invoked; the following required tools are not installed: nonexistent_tool (version 0.1), compose_text_param (version 0.0.1)", + ) + # should fail but return only the tool_id of non_existent tool as another version of compose_text_param is installed + invocation_response = self.workflow_populator.invoke_workflow( + workflow_id, history_id=history_id, assert_ok=False, request={"require_exact_tool_versions": False} + ) + self._assert_status_code_is(invocation_response, 400) + self.assertEqual( + invocation_response.json().get("err_msg"), + "Workflow was not invoked; the following required tools are not installed: nonexistent_tool", + ) From 3e8b6a70927b08f1eecf8374dc05be9b0fa5113b Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 10 Feb 2022 08:29:55 -0500 Subject: [PATCH 026/652] Force encoding of toolbox's tool_id routing to use '%20' instead of '+', since we ignore + because it's a valid character. --- client/src/components/Panels/ToolBox.vue | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/src/components/Panels/ToolBox.vue b/client/src/components/Panels/ToolBox.vue index 3af9756d9b7..5abc1cd1701 100644 --- a/client/src/components/Panels/ToolBox.vue +++ b/client/src/components/Panels/ToolBox.vue @@ -166,8 +166,9 @@ export default { } else if (tool.form_style === "regular") { evt.preventDefault(); const Galaxy = getGalaxyInstance(); + // encode spaces in tool.id Galaxy.router.push("/", { - tool_id: tool.id, + tool_id: tool.id.replace(/ /g, "%20"), version: tool.version, }); } From 5b4b981c586f5490ea1ebd9a24bd900725010e61 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Thu, 10 Feb 2022 17:04:09 +0100 Subject: [PATCH 027/652] remove test_run_workflow_with_missing_tool() from API tests --- lib/galaxy_test/api/test_workflows.py | 35 --------------------------- 1 file changed, 35 deletions(-) diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index 53ba9058ac1..2ec1936a698 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -919,41 +919,6 @@ steps: invocation = self._invocation_details(workflow_id, invocation_id) assert invocation["state"] == "scheduled", invocation - def test_run_workflow_with_missing_tool(self): - with self.dataset_populator.test_history() as history_id: - workflow_id = self._upload_yaml_workflow( - """ -class: GalaxyWorkflow -steps: - nonexistent: - tool_id: nonexistent_tool - tool_version: "0.1" - label: nonexistent - compose_text_param: - tool_id: multiple_versions - tool_version: 0.3 - label: multiple_versions -""" - ) - # should fail and return both tool ids since version 0.3 of multiple_versions does not exist - invocation_response = self.__invoke_workflow( - workflow_id, history_id=history_id, assert_ok=False, request={"require_exact_tool_versions": True} - ) - self._assert_status_code_is(invocation_response, 400) - self.assertEqual( - invocation_response.json().get("err_msg"), - "Workflow was not invoked; the following required tools are not installed: nonexistent_tool (version 0.1), multiple_versions (version 0.3)", - ) - # should fail but return only the tool_id of non_existent tool as another version of multiple_versions is installed - invocation_response = self.__invoke_workflow( - workflow_id, history_id=history_id, assert_ok=False, request={"require_exact_tool_versions": False} - ) - self._assert_status_code_is(invocation_response, 400) - self.assertEqual( - invocation_response.json().get("err_msg"), - "Workflow was not invoked; the following required tools are not installed: nonexistent_tool", - ) - @skip_without_tool("collection_creates_pair") def test_workflow_run_output_collections(self) -> None: with self.dataset_populator.test_history() as history_id: From f2a0ace199e8877ff8da5c51aa2833c59a626ddc Mon Sep 17 00:00:00 2001 From: Assunta DeSanto Date: Fri, 15 Oct 2021 13:07:50 -0400 Subject: [PATCH 028/652] first attempt at using infinite scroller in place of VS; not functional still not gathering data in the correct order; math sent to setScrollPos is not correct adding comments and updating ajax call to before -- requesting review from Dannon fixing a few errors pulling in Sergey's branch and removing erroneous console logs filtering duplicates cleaning some unused values Simple Axios Calls in Content Provider to slowly refactor pushing current hid to setScollPos as key, still some problems with payload recieved by InfiniteScroller removing merge conflict headers and a few comments/unused items removing some additional unused pieces to clean up files, and adding comments adding/modifying comments in simple provider removing top datasets from the cache payload, so duplicates are gone pushing payload from initial poll into data fixing issue where InfiniteScroller is created twice removing simple content provider no longer listening to scroll position; removes duplicate calls fixing some logic in getmorecontent, and adding delete functionality Add check for recently deleted adding polling on maxHID to look for new content removing commented out code and console logs adding watch for historyID, so contents change when switching to new history, and adding length check to polling, so brand new histories poll for new content changing to Vue Reactive Array; still some issues with status updates and updating not in current payload using near, using hid at the top of the scroller to determine current place Add flag to clear and reload history on dirty cache, revert changes to the cache --- .../History/ContentItem/Dataset/Dataset.vue | 1 + .../components/History/ContentOperations.vue | 16 +- client/src/components/History/History.vue | 47 ++--- .../components/History/InfiniteHistory.vue | 167 ++++++++++++++++++ .../ContentProvider/ContentProvider.js | 15 +- .../HistoryContentProvider/contentPayload.js | 3 +- .../History/caching/loadHistoryContents.js | 5 +- .../History/caching/monitorHistoryContent.js | 7 +- 8 files changed, 209 insertions(+), 52 deletions(-) create mode 100644 client/src/components/History/InfiniteHistory.vue diff --git a/client/src/components/History/ContentItem/Dataset/Dataset.vue b/client/src/components/History/ContentItem/Dataset/Dataset.vue index 6aca4d447a2..248a2f25f23 100644 --- a/client/src/components/History/ContentItem/Dataset/Dataset.vue +++ b/client/src/components/History/ContentItem/Dataset/Dataset.vue @@ -34,6 +34,7 @@ export default { methods: { async onDelete(opts = {}) { const ajaxResult = await deleteContent(this.item, opts); + this.$emit("deleted", this.item.hid); await cacheContent(ajaxResult); }, async onUnhide() { diff --git a/client/src/components/History/ContentOperations.vue b/client/src/components/History/ContentOperations.vue index 277b8b3bf54..6c8bb8590c2 100644 --- a/client/src/components/History/ContentOperations.vue +++ b/client/src/components/History/ContentOperations.vue @@ -194,6 +194,7 @@ export default { totalMatches: { type: Number, required: true }, expandedCount: { type: Number, required: false, default: 0 }, debug: { type: Boolean, default: false }, + setResetHistoryContents: {required: true} }, data() { return { @@ -212,6 +213,9 @@ export default { }, set(newVal) { this.$emit("update:params", newVal.clone()); + if (this.showContentFilters) { + this.setResetHistoryContents(true); + } }, }, @@ -242,15 +246,15 @@ export default { async unhideAll(evt) { await unhideAllHiddenContent(this.history); - this.$emit("manualReload"); + this.setResetHistoryContents(true); }, async deleteAllHidden(evt) { await deleteAllHiddenContent(this.history); - this.$emit("manualReload"); + this.setResetHistoryContents(true); }, async purgeAllDeleted(evt) { await purgeAllDeletedContent(this.history); - this.$emit("manualReload"); + this.setResetHistoryContents(true); }, // #endregion @@ -280,7 +284,7 @@ export default { console.log("operation results", results); } this.$emit("resetSelection"); - this.$emit("manualReload"); + this.setResetHistoryContents(true); }, // #endregion @@ -312,9 +316,9 @@ export default { await cacheContent({ ...dataset, visible: false }, true); }); this.$emit("resetSelection"); + // Force relaod since the source items should now be hidden + this.setResetHistoryContents(true); } - - // this.$emit("manualReload"); }, // #endregion diff --git a/client/src/components/History/History.vue b/client/src/components/History/History.vue index e4586eb72bf..06e26247340 100644 --- a/client/src/components/History/History.vue +++ b/client/src/components/History/History.vue @@ -5,7 +5,7 @@ :disable-poll="false" :debug="false" :debounce-period="500" - v-slot="{ loading, payload, manualReload, setScrollPos }"> + v-slot="{ loading, payload, setScrollPos, resetHistoryContents, setResetHistoryContents }"> @@ -61,18 +61,18 @@ + :page-size="params.pageSize" + :history-id="history.id" + :set-reset-history-contents="setResetHistoryContents" + :reset-history-contents="resetHistoryContents" />