Merge remote-tracking branch 'upstream/dev' into dev

client/src/components/Panels/ToolBoxWorkflow.vue

@@ -40,6 +40,7 @@
                     v-if="hasWorkflowSection"
                     :category="workflowSection"
                     :key="workflowSection.name"
+                    section-name="workflows"
                     operation-icon="fa fa-files-o"
                     operation-title="Insert individual steps."
                     :query-filter="query"

client/src/components/Workflow/Editor/Index.vue

@@ -243,6 +243,7 @@ export default {
             hasChanges: false,
             nodeIndex: 0,
             nodes: {},
+            requiresReindex: false, // track if node has been added or remove and backend may re-index nodes (hasChanges tracks a much more broad set of changes)
             datatypesMapper: null,
             datatypes: [],
             report: {},
@@ -347,6 +348,7 @@ export default {
         },
         onAdd(node) {
             this.nodes[node.id] = node;
+            this.requiresReindex = true;
         },
         onUpdate(node) {
             getModule({
@@ -367,6 +369,7 @@ export default {
             this.canvasManager.drawOverview();
             this.activeNode = null;
             this.hasChanges = true;
+            this.requiresReindex = true;
             showAttributes();
         },
         onEditSubworkflow(contentId) {
@@ -429,10 +432,20 @@ export default {
             node.onUnhighlight();
         },
         onLint() {
-            this._ensureParametersSet();
-            // See notes in Lint.vue about why refresh is needed.
-            this.$refs.lint.refresh();
-            showLint();
+            if (this.requiresReindex) {
+                const r = window.confirm(
+                    "Workflow steps have been added or removed since last save, the workflow needs to be saved before best practices can be analzyed. Save workflow?"
+                );
+                if (r == false) {
+                    return;
+                }
+                this.onSave(true);
+            } else {
+                this._ensureParametersSet();
+                // See notes in Lint.vue about why refresh is needed.
+                this.$refs.lint.refresh();
+                showLint();
+            }
         },
         onUpgrade() {
             this.attemptRefactor([{ action_type: "upgrade_all_steps" }]);

client/src/components/Workflow/Editor/Node.vue

@@ -321,7 +321,7 @@ export default {
         },
         setData(data) {
             this.config_form = data.config_form;
-            this.content_id = this.config_form?.id;
+            this.content_id = data.config_form?.id || data.content_id;
             this.tool_state = data.tool_state;
             this.errors = data.errors;
             this.annotation = data.annotation;

client/src/components/Workflow/Editor/Options.vue

@@ -95,6 +95,9 @@ export default {
         hasChanges: {
             type: Boolean,
         },
+        requiredReindex: {
+            type: Boolean,
+        },
     },
 };
 </script>

client/src/components/Workflow/Editor/modules/services.js

@@ -55,6 +55,7 @@ export async function saveWorkflow(workflow) {
             const { data } = await axios.put(`${getAppRoot()}api/workflows/${workflow.id}`, requestData);
             workflow.name = data.name;
             workflow.hasChanges = false;
+            workflow.requiresReindex = false;
             workflow.stored = true;
             workflow.version = data.version;
             workflow.annotation = data.annotation;

client/src/ui/search-input.js

@@ -37,7 +37,7 @@ function searchInput(parentNode, options) {
     // .................................................................... input rendering and events
     // visually clear the search, trigger an event, and call the callback
     function clearSearchInput(event) {
-        var $input = $(this).parent().children("input");
+        var $input = $(this).closest(".search-control").children("input");
         $input.val("").trigger("searchInput.clear").blur();
         options.onclear();
     }

lib/galaxy/selenium/navigation.yml

@@ -407,6 +407,9 @@ workflow_editor:
     tool_menu: '.toolMenuContainer'
     tool_menu_section_link: '.tool-menu-section-${section_name} a span'
     tool_menu_item_link: 'a.tool-menu-item-${item_name}'
+    workflow_link:
+      type: xpath
+      selector: '//a[contains(., "${workflow_title}")]'
 
     connect_icon: 'div.ui-form-element[tour_id="${name}"] .ui-form-connected-icon'
     collapse_icon: 'div.ui-form-element[tour_id="${name}"] .ui-form-collapsible-icon'

lib/galaxy/tools/execute.py

@@ -31,7 +31,7 @@ def execute(trans, tool, mapping_params, history, rerun_remap_job_id=None, colle
     Execute a tool and return object containing summary (output data, number of
     failures, etc...).
     """
-    if max_num_jobs:
+    if max_num_jobs is not None:
         assert invocation_step is not None
     if rerun_remap_job_id:
         assert invocation_step is None
@@ -93,7 +93,7 @@ def execute(trans, tool, mapping_params, history, rerun_remap_job_id=None, colle
     execution_slice = None
 
     for i, execution_slice in enumerate(execution_tracker.new_execution_slices()):
-        if max_num_jobs and jobs_executed >= max_num_jobs:
+        if max_num_jobs is not None and jobs_executed >= max_num_jobs:
             has_remaining_jobs = True
             break
         else:

lib/galaxy/tools/parameters/basic.py

@@ -1361,9 +1361,12 @@ class ColumnListParameter(SelectToolParameter):
         legal_values = self.get_column_list(trans, other_values)
 
         value = other_values.get(self.name)
-        if value is not None and value not in legal_values and self.is_file_empty(trans, other_values):
-            value = value if isinstance(value, list) else [value]
-            legal_values.extend(value)
+        if value is not None:
+            # There are cases where 'value' is a string of comma separated values. This ensures
+            # that it is converted into a list, with extra whitespace around items removed.
+            value = util.listify(value, do_strip=True)
+            if not set(value).issubset(set(legal_values)) and self.is_file_empty(trans, other_values):
+                legal_values.extend(value)
 
         return set(legal_values)
 

lib/galaxy/tools/toolbox/base.py

@@ -319,9 +319,13 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
 
     def get_shed_config_dict_by_filename(self, filename):
         filename = os.path.abspath(filename)
+        dynamic_tool_conf_paths = []
         for shed_config_dict in self._dynamic_tool_confs:
-            if shed_config_dict['config_filename'] == filename:
+            dynamic_tool_conf_path = os.path.abspath(shed_config_dict['config_filename'])
+            dynamic_tool_conf_paths.append(dynamic_tool_conf_path)
+            if dynamic_tool_conf_path == filename:
                 return shed_config_dict
+        log.warning("'{}' not among installable tool config files ({})".format(filename, ', '.join(dynamic_tool_conf_paths)))
         return None
 
     def update_shed_config(self, shed_conf):

lib/galaxy/webapps/galaxy/controllers/authnz.py

@@ -17,7 +17,7 @@ from galaxy.webapps.base.controller import JSAppLauncher
 
 log = logging.getLogger(__name__)
 
-PROVIDER_COOKIE_NAME = 'oidc-provider'
+PROVIDER_COOKIE_NAME = 'galaxy-oidc-provider'
 
 
 class OIDC(JSAppLauncher):

lib/galaxy/workflow/run.py

@@ -173,7 +173,12 @@ class WorkflowInvoker:
 
         remaining_steps = self.progress.remaining_steps()
         delayed_steps = False
+        max_jobs_per_iteration_reached = False
         for (step, workflow_invocation_step) in remaining_steps:
+            max_jobs_to_schedule = self.progress.maximum_jobs_to_schedule_or_none
+            if max_jobs_to_schedule is not None and max_jobs_to_schedule <= 0:
+                max_jobs_per_iteration_reached = True
+                break
             step_delayed = False
             step_timer = ExecutionTimer()
             try:
@@ -208,7 +213,7 @@ class WorkflowInvoker:
             if not step_delayed:
                 log.debug(f"Workflow step {step.id} of invocation {workflow_invocation.id} invoked {step_timer}")
 
-        if delayed_steps:
+        if delayed_steps or max_jobs_per_iteration_reached:
             state = model.WorkflowInvocation.states.READY
         else:
             state = model.WorkflowInvocation.states.SCHEDULED

lib/galaxy_test/api/test_workflows.py

@@ -1193,6 +1193,46 @@ steps:
             invocation_id = self.__invoke_workflow(history_id, workflow_id, inputs)
             self.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id)
 
+    @skip_without_tool('column_param')
+    def test_empty_file_data_column_specified(self):
+        # Regression test for https://github.com/galaxyproject/galaxy/pull/10981
+        with self.dataset_populator.test_history() as history_id:
+            self._run_jobs("""class: GalaxyWorkflow
+steps:
+  empty_output:
+    tool_id: empty_output
+    outputs:
+      out_file1:
+        change_datatype: tabular
+  column_param:
+    tool_id: column_param
+    in:
+      input1: empty_output/out_file1
+    state:
+      col: 2
+      col_names: 'B'
+""", history_id=history_id)
+
+    @skip_without_tool('column_param_list')
+    def test_comma_separated_columns(self):
+        # Regression test for https://github.com/galaxyproject/galaxy/pull/10981
+        with self.dataset_populator.test_history() as history_id:
+            self._run_jobs("""class: GalaxyWorkflow
+steps:
+  empty_output:
+    tool_id: empty_output
+    outputs:
+      out_file1:
+        change_datatype: tabular
+  column_param_list:
+    tool_id: column_param_list
+    in:
+      input1: empty_output/out_file1
+    state:
+      col: '2,3'
+      col_names: 'B'
+""", history_id=history_id)
+
     @skip_without_tool("mapper")
     @skip_without_tool("pileup")
     def test_workflow_metadata_validation_0(self):

lib/galaxy_test/selenium/test_workflow_editor.py

@@ -2,6 +2,7 @@ import json
 
 from galaxy_test.base.workflow_fixtures import (
     WORKFLOW_NESTED_SIMPLE,
+    WORKFLOW_OPTIONAL_TRUE_INPUT_COLLECTION,
     WORKFLOW_SIMPLE_CAT_TWICE,
     WORKFLOW_SIMPLE_MAPPING,
     WORKFLOW_WITH_INVALID_STATE,
@@ -358,6 +359,36 @@ steps:
         self.assert_modal_has_text("Using version '0.2' instead of version '0.0.1'")
         self.screenshot("workflow_editor_tool_upgrade")
 
+    @selenium_test
+    def test_editor_embed_workflow(self):
+        workflow_populator = self.workflow_populator
+        child_workflow_name = self._get_random_name()
+        workflow_populator.upload_yaml_workflow(WORKFLOW_OPTIONAL_TRUE_INPUT_COLLECTION, name=child_workflow_name)
+        parent_workflow_id = workflow_populator.upload_yaml_workflow("""class: GalaxyWorkflow
+inputs: []
+steps:
+  - tool_id: multiple_versions
+    tool_version: 0.1
+    label: multiple_versions
+    state:
+      foo: bar
+        """)
+        self.workflow_index_open()
+        self.workflow_index_click_option("Edit")
+        editor = self.components.workflow_editor
+        editor.canvas_body.wait_for_visible()
+        editor.tool_menu.wait_for_visible()
+        editor.tool_menu_section_link(section_name="workflows").wait_for_and_click()
+        editor.workflow_link(workflow_title=child_workflow_name).wait_for_and_click()
+        self.sleep_for(self.wait_types.UX_RENDER)
+        self.assert_has_changes_and_save()
+        self.sleep_for(self.wait_types.UX_RENDER)
+        workflow = self.workflow_populator.download_workflow(parent_workflow_id)
+        subworkflow_step = workflow['steps']['1']
+        assert subworkflow_step['name'] == child_workflow_name
+        assert subworkflow_step['type'] == 'subworkflow'
+        assert subworkflow_step['subworkflow']['a_galaxy_workflow'] == 'true'
+
     @selenium_test
     def test_editor_invalid_tool_state(self):
         workflow_populator = self.workflow_populator

pull_request_template.md

@@ -0,0 +1,19 @@
+## What did you do? 
+- [describe the proposed changes]
+- 
+
+
+## Why did you make this change?
+(Cite Issue number OR provide rationalization of changes if no issue exists)
+(If fixing a bug, please add any relevant error or traceback)
+
+
+## How to test the changes? 
+(select the most appropriate option; if the latter, provide steps for testing below)
+- [ ] I've included appropriate automated tests (https://docs.galaxyproject.org/en/latest/dev/writing_tests.html)
+- [ ] Instructions for manual testing are as follows:
+
+1. [add testing steps and prerequisites here if you didn't write automated tests covering all your changes]
+
+## For UI Components
+- [ ] I've included a screenshot of the changes

test/functional/tools/column_param_list.xml

@@ -0,0 +1,50 @@
+<tool id="column_param_list" name="Column Param List" version="1.0.0">
+    <command><![CDATA[
+cut -f '$col' '$input1' > '$output1' &&
+echo "col $col" > '$output2' &&
+echo "col_names $col_names" >> '$output2'
+    ]]></command>
+    <inputs>
+        <param name="input1" type="data" format="tabular" label="Input 1" />
+        <param name="col" type="data_column" multiple="true" value="1" data_ref="input1" label="Column to Use" />
+        <param name="col_names" type="data_column" data_ref="input1" use_header_names="true" label="Column to Use" />
+    </inputs>
+    <outputs>
+        <data name="output1" format="tabular" />
+        <data name="output2" format="txt" />
+    </outputs>
+    <tests>
+        <test>
+            <param name="input1" value="2.tabular" />
+            <param name="col" value="2" />
+            <param name="col_names" value="2" />
+            <output name="output1">
+                <assert_contents>
+                    <has_line line="68" />
+                </assert_contents>
+            </output>
+            <output name="output2">
+                <assert_contents>
+                    <has_line line="col 2" />
+                    <has_line line="col_names 2" />
+                </assert_contents>
+            </output>
+        </test>
+        <!-- test if non tabular data also creates entries by using the default
+            value (which is the 1st column, but empty if filling the options fails) -->
+        <test>
+            <param name="input1" value="1.bed" />
+            <output name="output1">
+                <assert_contents>
+                    <has_line line="chr1" />
+                </assert_contents>
+            </output>
+            <output name="output2">
+                <assert_contents>
+                    <has_line line="col 1" />
+                    <has_line line="col_names 1" />
+                </assert_contents>
+            </output>
+        </test>
+    </tests>
+</tool>

test/functional/tools/samples_tool_conf.xml

@@ -114,6 +114,7 @@
   <tool file="param_text_option.xml" />
   <tool file="column_param.xml" />
   <tool file="column_param_configfile.xml" />
+  <tool file="column_param_list.xml" />
   <tool file="column_multi_param.xml" />
   <tool file="hidden_param.xml" />
   <tool file="special_params.xml" />

test/integration/test_workflow_scheduling_options.py

@@ -61,7 +61,7 @@ class MaximumWorkflowJobsPerSchedulingIterationTestCase(integration_util.Integra
     def handle_galaxy_config_kwds(cls, config):
         config["maximum_workflow_jobs_per_scheduling_iteration"] = 1
 
-    def test(self):
+    def test_collection_explicit_and_implicit(self):
         workflow_id = self.workflow_populator.upload_yaml_workflow("""
 class: GalaxyWorkflow
 steps:
@@ -89,3 +89,40 @@ steps:
             self.workflow_populator.wait_for_workflow(history_id, workflow_id, invocation_id)
             self.dataset_populator.wait_for_history(history_id, assert_ok=True)
             self.assertEqual("a\nc\nb\nd\ne\ng\nf\nh\n", self.dataset_populator.get_history_dataset_content(history_id, hid=0))
+
+    def test_scheduling_rounds(self):
+        with self.dataset_populator.test_history() as history_id:
+            invocation_response = self.workflow_populator.run_workflow("""
+class: GalaxyWorkflow
+inputs:
+  input1: data
+  text_input: text
+steps:
+  first_cat:
+    tool_id: cat1
+    in:
+      input1: input1
+  second_cat:
+    tool_id: cat1
+    in:
+      input1: first_cat/out_file1
+  collection_creates_dynamic_list_of_pairs:
+    tool_id: collection_creates_dynamic_list_of_pairs
+    in:
+      file: second_cat/out_file1
+  count_multi_file:
+    tool_id: count_multi_file
+    in:
+      input1: collection_creates_dynamic_list_of_pairs/list_output
+outputs:
+  wf_output_1:
+    outputSource: collection_creates_dynamic_list_of_pairs/list_output
+""", test_data="""
+input1:
+  value: 1.fasta
+  type: File
+  name: fasta1
+text_input: foo
+""", history_id=history_id)
+            invocation = self._get("/invocations/{}".format(invocation_response.invocation_id)).json()
+            assert 'wf_output_1' in invocation['output_collections']