diff --git a/client/src/components/Panels/ToolBoxWorkflow.vue b/client/src/components/Panels/ToolBoxWorkflow.vue index 010f624233f..64e98721273 100644 --- a/client/src/components/Panels/ToolBoxWorkflow.vue +++ b/client/src/components/Panels/ToolBoxWorkflow.vue @@ -40,6 +40,7 @@ v-if="hasWorkflowSection" :category="workflowSection" :key="workflowSection.name" + section-name="workflows" operation-icon="fa fa-files-o" operation-title="Insert individual steps." :query-filter="query" diff --git a/client/src/components/Workflow/Editor/Index.vue b/client/src/components/Workflow/Editor/Index.vue index 942cea317fd..85ca6fa1ede 100644 --- a/client/src/components/Workflow/Editor/Index.vue +++ b/client/src/components/Workflow/Editor/Index.vue @@ -243,6 +243,7 @@ export default { hasChanges: false, nodeIndex: 0, nodes: {}, + requiresReindex: false, // track if node has been added or remove and backend may re-index nodes (hasChanges tracks a much more broad set of changes) datatypesMapper: null, datatypes: [], report: {}, @@ -347,6 +348,7 @@ export default { }, onAdd(node) { this.nodes[node.id] = node; + this.requiresReindex = true; }, onUpdate(node) { getModule({ @@ -367,6 +369,7 @@ export default { this.canvasManager.drawOverview(); this.activeNode = null; this.hasChanges = true; + this.requiresReindex = true; showAttributes(); }, onEditSubworkflow(contentId) { @@ -429,10 +432,20 @@ export default { node.onUnhighlight(); }, onLint() { - this._ensureParametersSet(); - // See notes in Lint.vue about why refresh is needed. - this.$refs.lint.refresh(); - showLint(); + if (this.requiresReindex) { + const r = window.confirm( + "Workflow steps have been added or removed since last save, the workflow needs to be saved before best practices can be analzyed. Save workflow?" + ); + if (r == false) { + return; + } + this.onSave(true); + } else { + this._ensureParametersSet(); + // See notes in Lint.vue about why refresh is needed. + this.$refs.lint.refresh(); + showLint(); + } }, onUpgrade() { this.attemptRefactor([{ action_type: "upgrade_all_steps" }]); diff --git a/client/src/components/Workflow/Editor/Node.vue b/client/src/components/Workflow/Editor/Node.vue index 908673387eb..fd1c7fd8ecc 100644 --- a/client/src/components/Workflow/Editor/Node.vue +++ b/client/src/components/Workflow/Editor/Node.vue @@ -321,7 +321,7 @@ export default { }, setData(data) { this.config_form = data.config_form; - this.content_id = this.config_form?.id; + this.content_id = data.config_form?.id || data.content_id; this.tool_state = data.tool_state; this.errors = data.errors; this.annotation = data.annotation; diff --git a/client/src/components/Workflow/Editor/Options.vue b/client/src/components/Workflow/Editor/Options.vue index f9f820c5a27..5a03c7f7b9a 100644 --- a/client/src/components/Workflow/Editor/Options.vue +++ b/client/src/components/Workflow/Editor/Options.vue @@ -95,6 +95,9 @@ export default { hasChanges: { type: Boolean, }, + requiredReindex: { + type: Boolean, + }, }, }; diff --git a/client/src/components/Workflow/Editor/modules/services.js b/client/src/components/Workflow/Editor/modules/services.js index 1da2ef2c86f..1783ffe5a51 100644 --- a/client/src/components/Workflow/Editor/modules/services.js +++ b/client/src/components/Workflow/Editor/modules/services.js @@ -55,6 +55,7 @@ export async function saveWorkflow(workflow) { const { data } = await axios.put(`${getAppRoot()}api/workflows/${workflow.id}`, requestData); workflow.name = data.name; workflow.hasChanges = false; + workflow.requiresReindex = false; workflow.stored = true; workflow.version = data.version; workflow.annotation = data.annotation; diff --git a/client/src/ui/search-input.js b/client/src/ui/search-input.js index a5ea8ef5de2..b8106b8143c 100644 --- a/client/src/ui/search-input.js +++ b/client/src/ui/search-input.js @@ -37,7 +37,7 @@ function searchInput(parentNode, options) { // .................................................................... input rendering and events // visually clear the search, trigger an event, and call the callback function clearSearchInput(event) { - var $input = $(this).parent().children("input"); + var $input = $(this).closest(".search-control").children("input"); $input.val("").trigger("searchInput.clear").blur(); options.onclear(); } diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml index 43dfa23dd32..4277f0723f1 100644 --- a/lib/galaxy/selenium/navigation.yml +++ b/lib/galaxy/selenium/navigation.yml @@ -407,6 +407,9 @@ workflow_editor: tool_menu: '.toolMenuContainer' tool_menu_section_link: '.tool-menu-section-${section_name} a span' tool_menu_item_link: 'a.tool-menu-item-${item_name}' + workflow_link: + type: xpath + selector: '//a[contains(., "${workflow_title}")]' connect_icon: 'div.ui-form-element[tour_id="${name}"] .ui-form-connected-icon' collapse_icon: 'div.ui-form-element[tour_id="${name}"] .ui-form-collapsible-icon' diff --git a/lib/galaxy/tools/execute.py b/lib/galaxy/tools/execute.py index 7468824f2b4..15b147af626 100644 --- a/lib/galaxy/tools/execute.py +++ b/lib/galaxy/tools/execute.py @@ -31,7 +31,7 @@ def execute(trans, tool, mapping_params, history, rerun_remap_job_id=None, colle Execute a tool and return object containing summary (output data, number of failures, etc...). """ - if max_num_jobs: + if max_num_jobs is not None: assert invocation_step is not None if rerun_remap_job_id: assert invocation_step is None @@ -93,7 +93,7 @@ def execute(trans, tool, mapping_params, history, rerun_remap_job_id=None, colle execution_slice = None for i, execution_slice in enumerate(execution_tracker.new_execution_slices()): - if max_num_jobs and jobs_executed >= max_num_jobs: + if max_num_jobs is not None and jobs_executed >= max_num_jobs: has_remaining_jobs = True break else: diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 97d20f66d0c..cec66a6eb14 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1361,9 +1361,12 @@ class ColumnListParameter(SelectToolParameter): legal_values = self.get_column_list(trans, other_values) value = other_values.get(self.name) - if value is not None and value not in legal_values and self.is_file_empty(trans, other_values): - value = value if isinstance(value, list) else [value] - legal_values.extend(value) + if value is not None: + # There are cases where 'value' is a string of comma separated values. This ensures + # that it is converted into a list, with extra whitespace around items removed. + value = util.listify(value, do_strip=True) + if not set(value).issubset(set(legal_values)) and self.is_file_empty(trans, other_values): + legal_values.extend(value) return set(legal_values) diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index f923a96af64..137815e5127 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -319,9 +319,13 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): def get_shed_config_dict_by_filename(self, filename): filename = os.path.abspath(filename) + dynamic_tool_conf_paths = [] for shed_config_dict in self._dynamic_tool_confs: - if shed_config_dict['config_filename'] == filename: + dynamic_tool_conf_path = os.path.abspath(shed_config_dict['config_filename']) + dynamic_tool_conf_paths.append(dynamic_tool_conf_path) + if dynamic_tool_conf_path == filename: return shed_config_dict + log.warning("'{}' not among installable tool config files ({})".format(filename, ', '.join(dynamic_tool_conf_paths))) return None def update_shed_config(self, shed_conf): diff --git a/lib/galaxy/webapps/galaxy/controllers/authnz.py b/lib/galaxy/webapps/galaxy/controllers/authnz.py index 5b7208622f5..f5960eaddf3 100644 --- a/lib/galaxy/webapps/galaxy/controllers/authnz.py +++ b/lib/galaxy/webapps/galaxy/controllers/authnz.py @@ -17,7 +17,7 @@ from galaxy.webapps.base.controller import JSAppLauncher log = logging.getLogger(__name__) -PROVIDER_COOKIE_NAME = 'oidc-provider' +PROVIDER_COOKIE_NAME = 'galaxy-oidc-provider' class OIDC(JSAppLauncher): diff --git a/lib/galaxy/workflow/run.py b/lib/galaxy/workflow/run.py index c261f23e7e2..f9402424fb2 100644 --- a/lib/galaxy/workflow/run.py +++ b/lib/galaxy/workflow/run.py @@ -173,7 +173,12 @@ class WorkflowInvoker: remaining_steps = self.progress.remaining_steps() delayed_steps = False + max_jobs_per_iteration_reached = False for (step, workflow_invocation_step) in remaining_steps: + max_jobs_to_schedule = self.progress.maximum_jobs_to_schedule_or_none + if max_jobs_to_schedule is not None and max_jobs_to_schedule <= 0: + max_jobs_per_iteration_reached = True + break step_delayed = False step_timer = ExecutionTimer() try: @@ -208,7 +213,7 @@ class WorkflowInvoker: if not step_delayed: log.debug(f"Workflow step {step.id} of invocation {workflow_invocation.id} invoked {step_timer}") - if delayed_steps: + if delayed_steps or max_jobs_per_iteration_reached: state = model.WorkflowInvocation.states.READY else: state = model.WorkflowInvocation.states.SCHEDULED diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index 4df8803c0bf..9e609f39477 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -1193,6 +1193,46 @@ steps: invocation_id = self.__invoke_workflow(history_id, workflow_id, inputs) self.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id) + @skip_without_tool('column_param') + def test_empty_file_data_column_specified(self): + # Regression test for https://github.com/galaxyproject/galaxy/pull/10981 + with self.dataset_populator.test_history() as history_id: + self._run_jobs("""class: GalaxyWorkflow +steps: + empty_output: + tool_id: empty_output + outputs: + out_file1: + change_datatype: tabular + column_param: + tool_id: column_param + in: + input1: empty_output/out_file1 + state: + col: 2 + col_names: 'B' +""", history_id=history_id) + + @skip_without_tool('column_param_list') + def test_comma_separated_columns(self): + # Regression test for https://github.com/galaxyproject/galaxy/pull/10981 + with self.dataset_populator.test_history() as history_id: + self._run_jobs("""class: GalaxyWorkflow +steps: + empty_output: + tool_id: empty_output + outputs: + out_file1: + change_datatype: tabular + column_param_list: + tool_id: column_param_list + in: + input1: empty_output/out_file1 + state: + col: '2,3' + col_names: 'B' +""", history_id=history_id) + @skip_without_tool("mapper") @skip_without_tool("pileup") def test_workflow_metadata_validation_0(self): diff --git a/lib/galaxy_test/selenium/test_workflow_editor.py b/lib/galaxy_test/selenium/test_workflow_editor.py index e5909de1149..64423feab0d 100644 --- a/lib/galaxy_test/selenium/test_workflow_editor.py +++ b/lib/galaxy_test/selenium/test_workflow_editor.py @@ -2,6 +2,7 @@ import json from galaxy_test.base.workflow_fixtures import ( WORKFLOW_NESTED_SIMPLE, + WORKFLOW_OPTIONAL_TRUE_INPUT_COLLECTION, WORKFLOW_SIMPLE_CAT_TWICE, WORKFLOW_SIMPLE_MAPPING, WORKFLOW_WITH_INVALID_STATE, @@ -358,6 +359,36 @@ steps: self.assert_modal_has_text("Using version '0.2' instead of version '0.0.1'") self.screenshot("workflow_editor_tool_upgrade") + @selenium_test + def test_editor_embed_workflow(self): + workflow_populator = self.workflow_populator + child_workflow_name = self._get_random_name() + workflow_populator.upload_yaml_workflow(WORKFLOW_OPTIONAL_TRUE_INPUT_COLLECTION, name=child_workflow_name) + parent_workflow_id = workflow_populator.upload_yaml_workflow("""class: GalaxyWorkflow +inputs: [] +steps: + - tool_id: multiple_versions + tool_version: 0.1 + label: multiple_versions + state: + foo: bar + """) + self.workflow_index_open() + self.workflow_index_click_option("Edit") + editor = self.components.workflow_editor + editor.canvas_body.wait_for_visible() + editor.tool_menu.wait_for_visible() + editor.tool_menu_section_link(section_name="workflows").wait_for_and_click() + editor.workflow_link(workflow_title=child_workflow_name).wait_for_and_click() + self.sleep_for(self.wait_types.UX_RENDER) + self.assert_has_changes_and_save() + self.sleep_for(self.wait_types.UX_RENDER) + workflow = self.workflow_populator.download_workflow(parent_workflow_id) + subworkflow_step = workflow['steps']['1'] + assert subworkflow_step['name'] == child_workflow_name + assert subworkflow_step['type'] == 'subworkflow' + assert subworkflow_step['subworkflow']['a_galaxy_workflow'] == 'true' + @selenium_test def test_editor_invalid_tool_state(self): workflow_populator = self.workflow_populator diff --git a/pull_request_template.md b/pull_request_template.md new file mode 100644 index 00000000000..2affb105a5f --- /dev/null +++ b/pull_request_template.md @@ -0,0 +1,19 @@ +## What did you do? +- [describe the proposed changes] +- + + +## Why did you make this change? +(Cite Issue number OR provide rationalization of changes if no issue exists) +(If fixing a bug, please add any relevant error or traceback) + + +## How to test the changes? +(select the most appropriate option; if the latter, provide steps for testing below) +- [ ] I've included appropriate automated tests (https://docs.galaxyproject.org/en/latest/dev/writing_tests.html) +- [ ] Instructions for manual testing are as follows: + +1. [add testing steps and prerequisites here if you didn't write automated tests covering all your changes] + +## For UI Components +- [ ] I've included a screenshot of the changes diff --git a/test/functional/tools/column_param_list.xml b/test/functional/tools/column_param_list.xml new file mode 100644 index 00000000000..a3c8d99e4f6 --- /dev/null +++ b/test/functional/tools/column_param_list.xml @@ -0,0 +1,50 @@ + + '$output1' && +echo "col $col" > '$output2' && +echo "col_names $col_names" >> '$output2' + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index b01cd8f42af..1a53ca33bf6 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -114,6 +114,7 @@ + diff --git a/test/integration/test_workflow_scheduling_options.py b/test/integration/test_workflow_scheduling_options.py index 50a2809554b..d8f24232413 100644 --- a/test/integration/test_workflow_scheduling_options.py +++ b/test/integration/test_workflow_scheduling_options.py @@ -61,7 +61,7 @@ class MaximumWorkflowJobsPerSchedulingIterationTestCase(integration_util.Integra def handle_galaxy_config_kwds(cls, config): config["maximum_workflow_jobs_per_scheduling_iteration"] = 1 - def test(self): + def test_collection_explicit_and_implicit(self): workflow_id = self.workflow_populator.upload_yaml_workflow(""" class: GalaxyWorkflow steps: @@ -89,3 +89,40 @@ steps: self.workflow_populator.wait_for_workflow(history_id, workflow_id, invocation_id) self.dataset_populator.wait_for_history(history_id, assert_ok=True) self.assertEqual("a\nc\nb\nd\ne\ng\nf\nh\n", self.dataset_populator.get_history_dataset_content(history_id, hid=0)) + + def test_scheduling_rounds(self): + with self.dataset_populator.test_history() as history_id: + invocation_response = self.workflow_populator.run_workflow(""" +class: GalaxyWorkflow +inputs: + input1: data + text_input: text +steps: + first_cat: + tool_id: cat1 + in: + input1: input1 + second_cat: + tool_id: cat1 + in: + input1: first_cat/out_file1 + collection_creates_dynamic_list_of_pairs: + tool_id: collection_creates_dynamic_list_of_pairs + in: + file: second_cat/out_file1 + count_multi_file: + tool_id: count_multi_file + in: + input1: collection_creates_dynamic_list_of_pairs/list_output +outputs: + wf_output_1: + outputSource: collection_creates_dynamic_list_of_pairs/list_output +""", test_data=""" +input1: + value: 1.fasta + type: File + name: fasta1 +text_input: foo +""", history_id=history_id) + invocation = self._get("/invocations/{}".format(invocation_response.invocation_id)).json() + assert 'wf_output_1' in invocation['output_collections']