Allow upload format 2 workflows to Galaxy.

lib/galaxy/config.py

@@ -382,6 +382,8 @@ class Configuration(object):
         # workflows built using these modules may not function in the
         # future.
         self.enable_beta_workflow_modules = string_as_bool(kwargs.get('enable_beta_workflow_modules', 'False'))
+        # Enable use of gxformat2 workflows.
+        self.enable_beta_workflow_format = string_as_bool(kwargs.get('enable_beta_workflow_format', 'False'))
         # These are not even beta - just experiments - don't use them unless
         # you want yours tools to be broken in the future.
         self.enable_beta_tool_formats = string_as_bool(kwargs.get('enable_beta_tool_formats', 'False'))

lib/galaxy/managers/workflows.py

@@ -38,6 +38,8 @@ from galaxy.workflow.modules import (
 )
 from galaxy.workflow.resources import get_resource_mapper_function
 from galaxy.workflow.steps import attach_ordered_steps
+from gxformat2.interface import ImporterGalaxyInterface
+from gxformat2.converter import python_to_workflow
 from .base import decode_id
 
 log = logging.getLogger(__name__)
@@ -235,6 +237,25 @@ class WorkflowContentsManager(UsesAnnotations):
         self.app = app
         self._resource_mapper_function = get_resource_mapper_function(app)
 
+    def normalize_workflow_format(self, as_dict):
+        """Process incoming workflow descriptions for consumption by other methods.
+
+        Currently this mostly means converting format 2 workflows into standard Galaxy
+        workflow JSON for consumption for the rest of this module. In the future we will
+        want to be a lot more percise about this - preserve the original description along
+        side the data model and apply updates in a way that largely preserves YAML structure
+        so workflows can be extracted.
+        """
+        workflow_class = as_dict.get("class", None)
+        if workflow_class == "GalaxyWorkflow":
+            if not self.app.config.enable_beta_workflow_format:
+                raise exceptions.ConfigDoesNotAllowException("Format2 workflows not enabled.")
+
+            # Format 2 Galaxy workflow.
+            galaxy_interface = Format2ConverterGalaxyInterface()
+            as_dict = python_to_workflow(as_dict, galaxy_interface, workflow_directory=None)
+        return as_dict
+
     def build_workflow_from_dict(
         self,
         trans,
@@ -1023,3 +1044,9 @@ class MissingToolsException(exceptions.MessageException):
     def __init__(self, workflow, errors):
         self.workflow = workflow
         self.errors = errors
+
+
+class Format2ConverterGalaxyInterface(ImporterGalaxyInterface):
+
+    def import_workflow(self, workflow, **kwds):
+        raise NotImplementedError("Direct format 2 import of nested workflows is not yet implemented, use bioblend client.")

lib/galaxy/webapps/galaxy/api/workflows.py

@@ -492,6 +492,7 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn
         stored_workflow = self.__get_stored_workflow(trans, id)
         workflow_dict = payload.get('workflow') or payload
         if workflow_dict:
+            workflow_dict = self.__normalize_workflow(workflow_dict)
             new_workflow_name = workflow_dict.get('name') or workflow_dict.get('name')
             if new_workflow_name and new_workflow_name != stored_workflow.name:
                 sanitized_name = sanitize_html(new_workflow_name)
@@ -572,6 +573,7 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn
             raise exceptions.MessageException("The data content does not appear to be a valid workflow.")
         if not data:
             raise exceptions.MessageException("The data content is missing.")
+        data = self.__normalize_workflow(data)
         workflow, missing_tool_tups = self._workflow_from_dict(trans, data, source=source)
         workflow = workflow.latest_workflow
         if workflow.has_errors:
@@ -584,6 +586,7 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn
 
     def __api_import_new_workflow(self, trans, payload, **kwd):
         data = payload['workflow']
+        data = self.__normalize_workflow(data)
         import_tools = util.string_as_bool(payload.get("import_tools", False))
         if import_tools and not trans.user_is_admin:
             raise exceptions.AdminRequiredException()
@@ -651,6 +654,9 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn
             'fill_defaults': fill_defaults,
         }
 
+    def __normalize_workflow(self, as_dict):
+        return self.workflow_contents_manager.normalize_workflow_format(as_dict)
+
     @expose_api
     def import_shared_workflow_deprecated(self, trans, payload, **kwd):
         """

lib/galaxy/webapps/galaxy/config_schema.yml

@@ -2100,6 +2100,13 @@ mapping:
           Enable beta workflow modules that should not yet be considered part of Galaxy's
           stable API.
 
+      enable_beta_workflow_format:
+        type: bool
+        default: false
+        required: false
+        desc: |
+          Enable import and export of workflows as Galaxy Format 2 workflows.
+
       force_beta_workflow_scheduled_min_steps:
         type: int
         default: 250

run.sh

@@ -38,6 +38,7 @@ if [ ! -z "$GALAXY_RUN_WITH_TEST_TOOLS" ];
 then
     export GALAXY_CONFIG_OVERRIDE_TOOL_CONFIG_FILE="test/functional/tools/samples_tool_conf.xml"
     export GALAXY_CONFIG_ENABLE_BETA_WORKFLOW_MODULES="true"
+    export GALAXY_CONFIG_ENABLE_BETA_WORKFLOW_FORMAT="true"
     export GALAXY_CONFIG_OVERRIDE_ENABLE_BETA_TOOL_FORMATS="true"
     export GALAXY_CONFIG_OVERRIDE_WEBHOOKS_DIR="test/functional/webhooks"
 fi

test/api/test_workflows_from_yaml.py

@@ -22,13 +22,13 @@ class WorkflowsFromYamlApiTestCase(BaseWorkflowsApiTestCase):
     def setUp(self):
         super(WorkflowsFromYamlApiTestCase, self).setUp()
 
-    def _upload_and_download(self, yaml_workflow):
-        workflow_id = self._upload_yaml_workflow(yaml_workflow)
+    def _upload_and_download(self, yaml_workflow, **kwds):
+        workflow_id = self._upload_yaml_workflow(yaml_workflow, **kwds)
         workflow = self._get("workflows/%s/download" % workflow_id).json()
         return workflow
 
     def test_simple_upload(self):
-        workflow = self._upload_and_download(WORKFLOW_SIMPLE_CAT_AND_RANDOM_LINES)
+        workflow = self._upload_and_download(WORKFLOW_SIMPLE_CAT_AND_RANDOM_LINES, client_convert=False)
 
         tool_count = {'random_lines1': 0, 'cat1': 0}
         input_found = False

test/base/driver_util.py

@@ -201,6 +201,7 @@ def setup_galaxy_config(
         cleanup_job='onsuccess',
         data_manager_config_file=data_manager_config_file,
         enable_beta_tool_formats=True,
+        enable_beta_workflow_format=True,
         expose_dataset_path=True,
         file_path=file_path,
         ftp_upload_purge=False,

test/base/populators.py

@@ -547,6 +547,8 @@ class BaseWorkflowPopulator(object):
         return upload_response
 
     def upload_yaml_workflow(self, has_yaml, **kwds):
+        client_convert = kwds.pop("client_convert", True)
+        kwds["convert"] = client_convert
         workflow = convert_and_import_workflow(has_yaml, galaxy_interface=self, **kwds)
         return workflow["id"]