Merge remote-tracking branch 'upstream/release_20.09' into release_20.09

client/src/components/LibraryFolder/TopToolbar/import-to-history/import-collection.js

@@ -91,11 +91,12 @@ var ImportCollectionModal = Backbone.View.extend({
         const new_history_name = this.modal.$("input[name=history_name]").val();
         if (new_history_name !== "") {
             this.createNewHistory(new_history_name)
-                .done((new_history) => {
+                .then((new_history) => {
                     Toast.success("History created");
                     this.collectionImport(collection_elements, new_history.id, new_history.name);
                 })
-                .fail((xhr, status, error) => {
+                .catch((error) => {
+                    console.error(error);
                     Toast.error("An error occurred.");
                 });
         } else {

client/src/components/Panels/ToolBoxWorkflow.vue

@@ -40,6 +40,7 @@
                 <tool-section
                     :category="workflowSection"
                     :key="workflowSection.name"
+                    section-name="workflows"
                     operation-icon="fa fa-files-o"
                     operation-title="Insert individual steps."
                     :query-filter="query"

client/src/components/Workflow/Editor/Node.vue

@@ -294,7 +294,7 @@ export default {
         },
         setData(data) {
             this.config_form = data.config_form;
-            this.content_id = this.config_form?.id;
+            this.content_id = data.config_form?.id || data.content_id;
             this.tool_state = data.tool_state;
             this.errors = data.errors;
             this.annotation = data.annotation;

client/src/mvc/ui/ui-file-source.js

@@ -45,7 +45,9 @@ var View = Backbone.View.extend({
     render: function () {
         const value = this._value;
         if (value) {
-            if (value.url !== this.$text.text()) {
+            if (typeof value == "string") {
+                this.$text.text(value);
+            } else if (value.url !== this.$text.text()) {
                 this.$text.text(value.url);
             }
         } else {
@@ -79,18 +81,27 @@ var View = Backbone.View.extend({
 
     /** Returns current value */
     _getValue: function () {
-        return this._value.url;
+        return this._value?.url;
     },
 
     /** Sets current value */
     _setValue: function (new_value) {
         if (new_value) {
             if (typeof new_value == "string") {
-                new_value = JSON.parse(new_value);
+                let parsed_value;
+                // if new_value is not a JSON, set it as String
+                try {
+                    parsed_value = JSON.parse(new_value);
+                } catch (e) {
+                    parsed_value = new_value;
+                } finally {
+                    new_value = parsed_value;
+                }
             }
             this._value = new_value;
             this.model.trigger("error", null);
             this.model.trigger("change");
+            this.trigger("change");
         }
     },
 });

doc/source/releases/20.09_announce.rst

@@ -76,6 +76,16 @@ To update an existing Galaxy repository run:
 
 See the `community hub <https://galaxyproject.org/develop/source-code/>`__ for additional details regarding the source code locations.
 
+Release Testing Team
+===========================================================
+
+A special thanks to the release testing team for testing many of the new features and reporting many bugs:
+
+- `Jayadev Joshi <https://github.com/jaidevjoshi83>`_
+- `Luke Sargent <https://github.com/luke-c-sargent>`_
+- `Marius van den Beek <https://github.com/mvdbeek>`_
+- `Sergey Golitsynskiy <https://github.com/ic4f>`_
+
 Release Notes
 ===========================================================
 

lib/galaxy/datatypes/binary.py

@@ -964,7 +964,7 @@ class Anndata(H5):
     MetadataElement(name="layers_count", default=0, desc="layers_count", readonly=True, visible=True, no_value=0)
     MetadataElement(name="layers_names", desc="layers_names", default=[], param=metadata.SelectParameter, multiple=True, readonly=True, no_value=None)
     MetadataElement(name="row_attrs_count", default=0, desc="row_attrs_count", readonly=True, visible=True, no_value=0)
-    MetadataElement(name="obs_names", desc="obs_names", default=[], param=metadata.SelectParameter, multiple=True, readonly=True, no_value=None)
+    MetadataElement(name="obs_names", desc="obs_names", default=[], multiple=True, readonly=True, no_value=None)
     MetadataElement(name="obs_layers", desc="obs_layers", default=[], param=metadata.SelectParameter, multiple=True, readonly=True, no_value=None)
     MetadataElement(name="obs_count", default=0, desc="obs_count", readonly=True, visible=True, no_value=0)
     MetadataElement(name="obs_size", default=-1, desc="obs_size", readonly=True, visible=True, no_value=0)

lib/galaxy/dependencies/__init__.py

@@ -220,7 +220,7 @@ class ConditionalDependencies:
         return 'webdav' in self.file_sources
 
     def check_watchdog(self):
-        install_set = {'auto', 'True', 'true', 'polling'}
+        install_set = {'auto', 'True', 'true', 'polling', True}
         return (self.config['watch_tools'] in install_set or
                 self.config['watch_tool_data_dir'] in install_set)
 

lib/galaxy/model/__init__.py

@@ -386,7 +386,7 @@ class User(Dictifiable, RepresentById):
         extra_user_preferences = self.preferences.get('extra_user_preferences')
         if extra_user_preferences:
             try:
-                data = json.loads(extra_user_preferences)
+                data.update(json.loads(extra_user_preferences))
             except Exception:
                 pass
         return data

lib/galaxy/selenium/navigation.yml

@@ -372,6 +372,9 @@ workflow_editor:
     tool_menu: '#workflow-tool-menu'
     tool_menu_section_link: '.tool-menu-section-${section_name} a span'
     tool_menu_item_link: 'a.tool-menu-item-${item_name}'
+    workflow_link:
+      type: xpath
+      selector: '//a[contains(., "${workflow_title}")]'
 
     connect_icon: 'div.ui-form-element[tour_id="${name}"] .ui-form-connected-icon'
     collapse_icon: 'div.ui-form-element[tour_id="${name}"] .ui-form-collapsible-icon'

lib/galaxy/tool_util/verify/__init__.py

@@ -371,4 +371,4 @@ def files_contains(file1, file2, attributes=None):
         if contains not in history_data:
             line_diff_count += 1
         if line_diff_count > lines_diff:
-            raise AssertionError("Failed to find '%s' in history data. (lines_diff=%i):\n" % (contains, lines_diff))
+            raise AssertionError("Failed to find '%s' in history data. (lines_diff=%i)" % (contains, lines_diff))

lib/galaxy/tools/__init__.py

@@ -1358,7 +1358,7 @@ class Tool(Dictifiable):
     def populate_resource_parameters(self, tool_source):
         root = getattr(tool_source, 'root', None)
         if root is not None and hasattr(self.app, 'job_config') and hasattr(self.app.job_config, 'get_tool_resource_xml'):
-            resource_xml = self.app.job_config.get_tool_resource_xml(root.get('id'), self.tool_type)
+            resource_xml = self.app.job_config.get_tool_resource_xml(root.get('id', '').lower(), self.tool_type)
             if resource_xml is not None:
                 inputs = root.find('inputs')
                 if inputs is None:

lib/galaxy/tools/evaluation.py

@@ -1,6 +1,7 @@
 import json
 import logging
 import os
+import shlex
 import tempfile
 
 
@@ -497,8 +498,7 @@ class ToolEvaluator:
             executable = command_line.split()[0]
             tool_dir = os.path.abspath(self.tool.tool_dir)
             abs_executable = os.path.join(tool_dir, executable)
-            command_line = command_line.replace(executable, abs_executable, 1)
-            command_line = interpreter + " " + command_line
+            command_line = command_line.replace(executable, "{} {}".format(interpreter, shlex.quote(abs_executable)), 1)
         self.command_line = command_line
 
     def __build_config_files(self):

lib/galaxy/tools/execute.py

@@ -34,7 +34,7 @@ def execute(trans, tool, mapping_params, history, rerun_remap_job_id=None, colle
     Execute a tool and return object containing summary (output data, number of
     failures, etc...).
     """
-    if max_num_jobs:
+    if max_num_jobs is not None:
         assert invocation_step is not None
     if rerun_remap_job_id:
         assert invocation_step is None
@@ -96,7 +96,7 @@ def execute(trans, tool, mapping_params, history, rerun_remap_job_id=None, colle
     execution_slice = None
 
     for i, execution_slice in enumerate(execution_tracker.new_execution_slices()):
-        if max_num_jobs and jobs_executed >= max_num_jobs:
+        if max_num_jobs is not None and jobs_executed >= max_num_jobs:
             has_remaining_jobs = True
             break
         else:

lib/galaxy/tools/toolbox/base.py

@@ -264,9 +264,13 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
 
     def get_shed_config_dict_by_filename(self, filename):
         filename = os.path.abspath(filename)
+        dynamic_tool_conf_paths = []
         for shed_config_dict in self._dynamic_tool_confs:
-            if shed_config_dict['config_filename'] == filename:
+            dynamic_tool_conf_path = os.path.abspath(shed_config_dict['config_filename'])
+            dynamic_tool_conf_paths.append(dynamic_tool_conf_path)
+            if dynamic_tool_conf_path == filename:
                 return shed_config_dict
+        log.warning("'{}' not among installable tool config files ({})".format(filename, ', '.join(dynamic_tool_conf_paths)))
         return None
 
     def update_shed_config(self, shed_conf):

lib/galaxy/webapps/galaxy/controllers/authnz.py

@@ -16,7 +16,7 @@ from galaxy.webapps.base.controller import JSAppLauncher
 
 log = logging.getLogger(__name__)
 
-PROVIDER_COOKIE_NAME = 'oidc-provider'
+PROVIDER_COOKIE_NAME = 'galaxy-oidc-provider'
 
 
 class OIDC(JSAppLauncher):

lib/galaxy/workflow/run.py

@@ -173,7 +173,12 @@ class WorkflowInvoker:
 
         remaining_steps = self.progress.remaining_steps()
         delayed_steps = False
+        max_jobs_per_iteration_reached = False
         for (step, workflow_invocation_step) in remaining_steps:
+            max_jobs_to_schedule = self.progress.maximum_jobs_to_schedule_or_none
+            if max_jobs_to_schedule is not None and max_jobs_to_schedule <= 0:
+                max_jobs_per_iteration_reached = True
+                break
             step_delayed = False
             step_timer = ExecutionTimer()
             try:
@@ -208,7 +213,7 @@ class WorkflowInvoker:
             if not step_delayed:
                 log.debug("Workflow step {} of invocation {} invoked {}".format(step.id, workflow_invocation.id, step_timer))
 
-        if delayed_steps:
+        if delayed_steps or max_jobs_per_iteration_reached:
             state = model.WorkflowInvocation.states.READY
         else:
             state = model.WorkflowInvocation.states.SCHEDULED

lib/galaxy_test/selenium/test_workflow_editor.py

@@ -2,6 +2,7 @@ import json
 
 from galaxy_test.base.workflow_fixtures import (
     WORKFLOW_NESTED_SIMPLE,
+    WORKFLOW_OPTIONAL_TRUE_INPUT_COLLECTION,
     WORKFLOW_SIMPLE_CAT_TWICE,
     WORKFLOW_SIMPLE_MAPPING,
     WORKFLOW_WITH_INVALID_STATE,
@@ -331,6 +332,36 @@ steps:
         self.assert_modal_has_text("Using version '0.2' instead of version '0.0.1'")
         self.screenshot("workflow_editor_tool_upgrade")
 
+    @selenium_test
+    def test_editor_embed_workflow(self):
+        workflow_populator = self.workflow_populator
+        child_workflow_name = self._get_random_name()
+        workflow_populator.upload_yaml_workflow(WORKFLOW_OPTIONAL_TRUE_INPUT_COLLECTION, name=child_workflow_name)
+        parent_workflow_id = workflow_populator.upload_yaml_workflow("""class: GalaxyWorkflow
+inputs: []
+steps:
+  - tool_id: multiple_versions
+    tool_version: 0.1
+    label: multiple_versions
+    state:
+      foo: bar
+        """)
+        self.workflow_index_open()
+        self.workflow_index_click_option("Edit")
+        editor = self.components.workflow_editor
+        editor.canvas_body.wait_for_visible()
+        editor.tool_menu.wait_for_visible()
+        editor.tool_menu_section_link(section_name="workflows").wait_for_and_click()
+        editor.workflow_link(workflow_title=child_workflow_name).wait_for_and_click()
+        self.sleep_for(self.wait_types.UX_RENDER)
+        self.assert_has_changes_and_save()
+        self.sleep_for(self.wait_types.UX_RENDER)
+        workflow = self.workflow_populator.download_workflow(parent_workflow_id)
+        subworkflow_step = workflow['steps']['1']
+        assert subworkflow_step['name'] == child_workflow_name
+        assert subworkflow_step['type'] == 'subworkflow'
+        assert subworkflow_step['subworkflow']['a_galaxy_workflow'] == 'true'
+
     @selenium_test
     def test_editor_invalid_tool_state(self):
         workflow_populator = self.workflow_populator

packages/test.sh

@@ -14,6 +14,7 @@ TEST_ENV_DIR=${TEST_ENV_DIR:-$(mktemp -d -t gxpkgtestenvXXXXXX)}
 
 virtualenv -p "$TEST_PYTHON" "$TEST_ENV_DIR"
 . "${TEST_ENV_DIR}/bin/activate"
+pip install --upgrade pip setuptools wheel
 
 # ensure ordered by dependency dag
 PACKAGE_DIRS=(

test-data/bcf_index_metadata_test.txt

@@ -1 +1 @@
-gzip compressed data, extra field
+Blocked GNU Zip Format (BGZF; gzip compatible)

test/functional/tools/metadata_bcf.xml

@@ -1,5 +1,10 @@
 <tool id="metadata_bcf" name="metadata_BCF" version="1.0.0">
-    <command>file "${input_bcf.metadata.bcf_index}" &gt; "${output_of_input_metadata}"</command>
+    <requirements>
+        <requirement type="package" version="5.39">file</requirement>
+    </requirements>
+    <command><![CDATA[
+file '${input_bcf.metadata.bcf_index}' > '${output_of_input_metadata}'
+    ]]></command>
     <inputs>
       <param name="input_bcf" type="data" format="bcf" label="BCF File"/>
     </inputs>

test/integration/test_workflow_scheduling_options.py

@@ -61,7 +61,7 @@ class MaximumWorkflowJobsPerSchedulingIterationTestCase(integration_util.Integra
     def handle_galaxy_config_kwds(cls, config):
         config["maximum_workflow_jobs_per_scheduling_iteration"] = 1
 
-    def test(self):
+    def test_collection_explicit_and_implicit(self):
         workflow_id = self.workflow_populator.upload_yaml_workflow("""
 class: GalaxyWorkflow
 steps:
@@ -89,3 +89,40 @@ steps:
             self.workflow_populator.wait_for_workflow(history_id, workflow_id, invocation_id)
             self.dataset_populator.wait_for_history(history_id, assert_ok=True)
             self.assertEqual("a\nc\nb\nd\ne\ng\nf\nh\n", self.dataset_populator.get_history_dataset_content(history_id, hid=0))
+
+    def test_scheduling_rounds(self):
+        with self.dataset_populator.test_history() as history_id:
+            invocation_response = self.workflow_populator.run_workflow("""
+class: GalaxyWorkflow
+inputs:
+  input1: data
+  text_input: text
+steps:
+  first_cat:
+    tool_id: cat1
+    in:
+      input1: input1
+  second_cat:
+    tool_id: cat1
+    in:
+      input1: first_cat/out_file1
+  collection_creates_dynamic_list_of_pairs:
+    tool_id: collection_creates_dynamic_list_of_pairs
+    in:
+      file: second_cat/out_file1
+  count_multi_file:
+    tool_id: count_multi_file
+    in:
+      input1: collection_creates_dynamic_list_of_pairs/list_output
+outputs:
+  wf_output_1:
+    outputSource: collection_creates_dynamic_list_of_pairs/list_output
+""", test_data="""
+input1:
+  value: 1.fasta
+  type: File
+  name: fasta1
+text_input: foo
+""", history_id=history_id)
+            invocation = self._get("/invocations/{}".format(invocation_response.invocation_id)).json()
+            assert 'wf_output_1' in invocation['output_collections']