From ebbfb0b765f6ecd1b2dc26ce9915642abf55cde8 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 16 Sep 2022 18:56:35 +0200
Subject: [PATCH 1/9] Submit workflow input paramters as inputs in expanded
 tool form

This fixes recording optional data input module use in the expanded tool
form (and is closer to how we want to use the API).

Fixes https://github.com/galaxyproject/galaxy/issues/14611
---
 client/src/components/Workflow/Run/WorkflowRunForm.vue | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 67669e38790..84db753a236 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -93,6 +93,7 @@ export default {
             stepValidations: {},
             stepScrollTo: {},
             wpData: {},
+            inputs: {},
             historyData: {},
             useCachedJobs: false,
             historyInputs: [
@@ -155,8 +156,7 @@ export default {
             return [];
         },
         onDefaultStepInputs(stepId, data) {
-            this.stepData[stepId] = data;
-            this.stepData = Object.assign({}, this.stepData);
+            this.inputs[stepId] = data.input;
         },
         onToolStepInputs(stepId, data) {
             this.stepData[stepId] = data;
@@ -201,6 +201,7 @@ export default {
                 resource_params: this.resourceData,
                 replacement_params: this.wpData,
                 use_cached_job: this.useCachedJobs,
+                inputs: this.inputs,
                 parameters: parameters,
                 // Tool form will submit flat maps for each parameter
                 // (e.g. "repeat_0|cond|param": "foo" instead of nested

From eacf1f9b5cf8018a5ba4e794948e8a7288b6ecf7 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 16 Sep 2022 20:50:30 +0200
Subject: [PATCH 2/9] Render subworkflow step with replacable form param
 component

Tool and subworkflow steps can have replacement parameters
---
 client/src/components/Workflow/Run/WorkflowRunForm.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 84db753a236..b12355b132e 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -37,7 +37,7 @@
                 </FormCard>
                 <div v-for="step in model.steps" :key="step.index">
                     <WorkflowRunToolStep
-                        v-if="step.step_type == 'tool'"
+                        v-if="step.step_type == 'tool' || step.step_type == 'subworkflow'"
                         :model="step"
                         :replace-params="getReplaceParams(step.inputs)"
                         :validation-scroll-to="getValidationScrollTo(step.index)"

From a1e446a3e9e1ca3672ffb1de7a5d848bb4190860 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 16 Sep 2022 21:09:54 +0200
Subject: [PATCH 3/9] Rename WorkflowRunToolStep to WorkflowRunDefaultStep,
 WorkflowRunDefaultStep to WorkflowInputStep

I think these are better names that describe what they are used for.
---
 .../Workflow/Run/WorkflowRunDefaultStep.vue   |  80 +++++++++-----
 .../Workflow/Run/WorkflowRunForm.vue          |   8 +-
 .../Workflow/Run/WorkflowRunInputStep.vue     |  83 +++++++++++++++
 .../Workflow/Run/WorkflowRunToolStep.vue      | 100 ------------------
 .../WorkflowInvocationStep.vue                |  14 +--
 5 files changed, 141 insertions(+), 144 deletions(-)
 create mode 100644 client/src/components/Workflow/Run/WorkflowRunInputStep.vue
 delete mode 100644 client/src/components/Workflow/Run/WorkflowRunToolStep.vue

diff --git a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
index ba49c2c49ce..beae0371d3e 100644
--- a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
@@ -2,13 +2,19 @@
     <div :step-label="model.step_label">
         <FormCard :title="model.fixed_title" :icon="icon" :collapsible="true" :expanded.sync="expanded">
             <template v-slot:body>
+                <FormMessage :message="errorText" variant="danger" :persistent="true" />
                 <FormDisplay
-                    v-if="hasInputs"
-                    :inputs="inputs"
+                    :inputs="modelInputs"
+                    :sustain-repeats="true"
+                    :sustain-conditionals="true"
+                    :replace-params="replaceParams"
                     :validation-scroll-to="validationScrollTo"
+                    collapsed-enable-text="Edit"
+                    collapsed-enable-icon="fa fa-edit"
+                    collapsed-disable-text="Undo"
+                    collapsed-disable-icon="fa fa-undo"
                     @onChange="onChange"
                     @onValidation="onValidation" />
-                <div v-else class="py-2">No options available.</div>
             </template>
         </FormCard>
     </div>
@@ -17,52 +23,43 @@
 <script>
 import WorkflowIcons from "components/Workflow/icons";
 import FormDisplay from "components/Form/FormDisplay";
+import FormMessage from "components/Form/FormMessage";
 import FormCard from "components/Form/FormCard";
+import { visitInputs } from "components/Form/utilities";
+import { getTool } from "./services";
 
 export default {
     components: {
         FormDisplay,
         FormCard,
+        FormMessage,
     },
     props: {
         model: {
             type: Object,
             required: true,
         },
+        replaceParams: {
+            type: Object,
+            default: null,
+        },
         validationScrollTo: {
             type: Array,
             required: true,
         },
+        historyId: {
+            type: String,
+            default: null,
+        },
     },
     data() {
         return {
             expanded: this.model.expanded,
+            errorText: null,
+            modelIndex: {},
+            modelInputs: this.model.inputs,
         };
     },
-    computed: {
-        icon() {
-            return WorkflowIcons[this.model.step_type];
-        },
-        isSimpleInput() {
-            return (
-                this.model.step_type.startsWith("data_input") ||
-                this.model.step_type.startsWith("data_collection_input")
-            );
-        },
-        inputs() {
-            this.model.inputs.forEach((input) => {
-                input.flavor = "module";
-                input.hide_label = this.isSimpleInput;
-            });
-            if (this.model.inputs && this.model.inputs.length > 0) {
-                return this.model.inputs;
-            }
-            return [];
-        },
-        hasInputs() {
-            return this.inputs.length > 0;
-        },
-    },
     watch: {
         validationScrollTo() {
             if (this.validationScrollTo.length > 0) {
@@ -70,8 +67,35 @@ export default {
             }
         },
     },
+    computed: {
+        icon() {
+            return WorkflowIcons[this.model.step_type];
+        },
+    },
     methods: {
-        onChange(data) {
+        onCreateIndex() {
+            this.modelIndex = {};
+            visitInputs(this.modelInputs, (input, name) => {
+                this.modelIndex[name] = input;
+            });
+        },
+        onChange(data, refreshRequest) {
+            if (refreshRequest) {
+                getTool(this.model.id, this.model.version, data, this.historyId).then(
+                    (newModel) => {
+                        this.onCreateIndex();
+                        visitInputs(newModel.inputs, (newInput, name) => {
+                            const input = this.modelIndex[name];
+                            input.options = newInput.options;
+                            input.textable = newInput.textable;
+                        });
+                        this.modelInputs = JSON.parse(JSON.stringify(this.modelInputs));
+                    },
+                    (errorText) => {
+                        this.errorText = errorText;
+                    }
+                );
+            }
             this.$emit("onChange", this.model.index, data);
         },
         onValidation(validation) {
diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index b12355b132e..1fea4c76115 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -36,7 +36,7 @@
                     </template>
                 </FormCard>
                 <div v-for="step in model.steps" :key="step.index">
-                    <WorkflowRunToolStep
+                    <WorkflowRunDefaultStep
                         v-if="step.step_type == 'tool' || step.step_type == 'subworkflow'"
                         :model="step"
                         :replace-params="getReplaceParams(step.inputs)"
@@ -44,7 +44,7 @@
                         :history-id="currentHistoryId"
                         @onChange="onToolStepInputs"
                         @onValidation="onValidation" />
-                    <WorkflowRunDefaultStep
+                    <WorkflowRunInputStep
                         v-else
                         :model="step"
                         :validation-scroll-to="getValidationScrollTo(step.index)"
@@ -64,7 +64,7 @@ import FormCard from "components/Form/FormCard";
 import FormElement from "components/Form/FormElement";
 import UserHistories from "components/providers/UserHistories";
 import WorkflowRunDefaultStep from "./WorkflowRunDefaultStep";
-import WorkflowRunToolStep from "./WorkflowRunToolStep";
+import WorkflowRunInputStep from "./WorkflowRunInputStep";
 import { allowCachedJobs } from "components/Tool/utilities";
 import { getReplacements } from "./model";
 import { invokeWorkflow } from "./services";
@@ -78,7 +78,7 @@ export default {
         FormElement,
         UserHistories,
         WorkflowRunDefaultStep,
-        WorkflowRunToolStep,
+        WorkflowRunInputStep,
     },
     props: {
         model: {
diff --git a/client/src/components/Workflow/Run/WorkflowRunInputStep.vue b/client/src/components/Workflow/Run/WorkflowRunInputStep.vue
new file mode 100644
index 00000000000..1c27fe9fac8
--- /dev/null
+++ b/client/src/components/Workflow/Run/WorkflowRunInputStep.vue
@@ -0,0 +1,83 @@
+<template>
+    <div :step-label="model.step_label">
+        <FormCard :title="model.fixed_title" :icon="icon" :collapsible="true" :expanded.sync="expanded">
+            <template v-slot:body>
+                <FormDisplay
+                    v-if="hasInputs"
+                    :inputs="inputs"
+                    :validation-scroll-to="validationScrollTo"
+                    @onChange="onChange"
+                    @onValidation="onValidation" />
+                <div v-else class="py-2">No options available.</div>
+            </template>
+        </FormCard>
+    </div>
+</template>
+
+<script>
+import WorkflowIcons from "components/Workflow/icons";
+import FormDisplay from "components/Form/FormDisplay";
+import FormCard from "components/Form/FormCard";
+
+export default {
+    components: {
+        FormDisplay,
+        FormCard,
+    },
+    props: {
+        model: {
+            type: Object,
+            required: true,
+        },
+        validationScrollTo: {
+            type: Array,
+            required: true,
+        },
+    },
+    data() {
+        return {
+            expanded: this.model.expanded,
+        };
+    },
+    computed: {
+        icon() {
+            return WorkflowIcons[this.model.step_type];
+        },
+        isSimpleInput() {
+            return (
+                this.model.step_type.startsWith("data_input") ||
+                this.model.step_type.startsWith("data_collection_input")
+            );
+        },
+        inputs() {
+            this.model.inputs.forEach((input) => {
+                input.flavor = "module";
+                input.hide_label = this.isSimpleInput;
+            });
+            if (this.model.inputs && this.model.inputs.length > 0) {
+                return this.model.inputs;
+            }
+            return [];
+        },
+        hasInputs() {
+            return this.inputs.length > 0;
+        },
+    },
+    watch: {
+        validationScrollTo() {
+            if (this.validationScrollTo.length > 0) {
+                this.expanded = true;
+            }
+        },
+    },
+    methods: {
+        onChange(data) {
+            console.log("emitting default change", data);
+            this.$emit("onChange", this.model.index, data);
+        },
+        onValidation(validation) {
+            this.$emit("onValidation", this.model.index, validation);
+        },
+    },
+};
+</script>
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
deleted file mode 100644
index a90e575aa0b..00000000000
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ /dev/null
@@ -1,100 +0,0 @@
-<template>
-    <div :step-label="model.step_label">
-        <FormCard :title="model.fixed_title" icon="fa-wrench" :collapsible="true" :expanded.sync="expanded">
-            <template v-slot:body>
-                <FormMessage :message="errorText" variant="danger" :persistent="true" />
-                <FormDisplay
-                    :inputs="modelInputs"
-                    :sustain-repeats="true"
-                    :sustain-conditionals="true"
-                    :replace-params="replaceParams"
-                    :validation-scroll-to="validationScrollTo"
-                    collapsed-enable-text="Edit"
-                    collapsed-enable-icon="fa fa-edit"
-                    collapsed-disable-text="Undo"
-                    collapsed-disable-icon="fa fa-undo"
-                    @onChange="onChange"
-                    @onValidation="onValidation" />
-            </template>
-        </FormCard>
-    </div>
-</template>
-
-<script>
-import FormDisplay from "components/Form/FormDisplay";
-import FormMessage from "components/Form/FormMessage";
-import FormCard from "components/Form/FormCard";
-import { visitInputs } from "components/Form/utilities";
-import { getTool } from "./services";
-
-export default {
-    components: {
-        FormDisplay,
-        FormCard,
-        FormMessage,
-    },
-    props: {
-        model: {
-            type: Object,
-            required: true,
-        },
-        replaceParams: {
-            type: Object,
-            default: null,
-        },
-        validationScrollTo: {
-            type: Array,
-            required: true,
-        },
-        historyId: {
-            type: String,
-            default: null,
-        },
-    },
-    data() {
-        return {
-            expanded: this.model.expanded,
-            errorText: null,
-            modelIndex: {},
-            modelInputs: this.model.inputs,
-        };
-    },
-    watch: {
-        validationScrollTo() {
-            if (this.validationScrollTo.length > 0) {
-                this.expanded = true;
-            }
-        },
-    },
-    methods: {
-        onCreateIndex() {
-            this.modelIndex = {};
-            visitInputs(this.modelInputs, (input, name) => {
-                this.modelIndex[name] = input;
-            });
-        },
-        onChange(data, refreshRequest) {
-            if (refreshRequest) {
-                getTool(this.model.id, this.model.version, data, this.historyId).then(
-                    (newModel) => {
-                        this.onCreateIndex();
-                        visitInputs(newModel.inputs, (newInput, name) => {
-                            const input = this.modelIndex[name];
-                            input.options = newInput.options;
-                            input.textable = newInput.textable;
-                        });
-                        this.modelInputs = JSON.parse(JSON.stringify(this.modelInputs));
-                    },
-                    (errorText) => {
-                        this.errorText = errorText;
-                    }
-                );
-            }
-            this.$emit("onChange", this.model.index, data);
-        },
-        onValidation(validation) {
-            this.$emit("onValidation", this.model.index, validation);
-        },
-    },
-};
-</script>
diff --git a/client/src/components/WorkflowInvocationState/WorkflowInvocationStep.vue b/client/src/components/WorkflowInvocationState/WorkflowInvocationStep.vue
index 26df5c7301f..d76ddaf1a19 100644
--- a/client/src/components/WorkflowInvocationState/WorkflowInvocationStep.vue
+++ b/client/src/components/WorkflowInvocationState/WorkflowInvocationStep.vue
@@ -86,6 +86,7 @@
 <script>
 import { mapCacheActions } from "vuex-cache";
 import { mapGetters, mapActions } from "vuex";
+import WorkflowIcons from "components/Workflow/icons";
 import JobStep from "./JobStep";
 import ParameterStep from "./ParameterStep";
 import GenericHistoryItem from "components/History/Content/GenericItem";
@@ -134,18 +135,7 @@ export default {
             return ["data_input", "data_collection_input"].includes(this.workflowStepType);
         },
         stepIcon() {
-            switch (this.workflowStepType) {
-                case "data_input":
-                    return "fa-file";
-                case "data_collection_input":
-                    return "fa-folder-o";
-                case "parameter_input":
-                    return "fa-pencil";
-                case "subworkflow":
-                    return "fa-tasks";
-                default:
-                    return "fa-wrench";
-            }
+            return WorkflowIcons[this.workflowStepType];
         },
         stepLabel() {
             return this.labelForWorkflowStep(this.workflowStep.id);

From 53154ad2188f457fe70726473c2ecd39f48206a0 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 19 Sep 2022 17:21:22 +0200
Subject: [PATCH 4/9] Improve logging when set_outputs_for_inputs fails

---
 lib/galaxy/model/__init__.py | 4 +++-
 lib/galaxy/workflow/run.py   | 4 +---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index ca9405a6451..0fff633aede 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -7212,7 +7212,9 @@ class WorkflowStep(Base, RepresentById):
         copied_step.workflow_outputs = copy_list(self.workflow_outputs, copied_step)
 
     def log_str(self):
-        return "WorkflowStep[index=%d,type=%s]" % (self.order_index, self.type)
+        return (
+            f"WorkflowStep[index={self.order_index},type={self.type},label={self.label},uuid={self.uuid},id={self.id}]"
+        )
 
     def clear_module_extras(self):
         # the module code adds random dynamic state to the step, this
diff --git a/lib/galaxy/workflow/run.py b/lib/galaxy/workflow/run.py
index ac5af627826..5815ee0438a 100644
--- a/lib/galaxy/workflow/run.py
+++ b/lib/galaxy/workflow/run.py
@@ -391,9 +391,7 @@ class WorkflowProgress:
                 if default_value:
                     outputs["output"] = default_value
                 else:
-                    template = "Step with id %s not found in inputs_step_id (%s)"
-                    message = template % (step.log_str(), self.inputs_by_step_id)
-                    raise ValueError(message)
+                    raise ValueError(f"{step.log_str()} not found in inputs_step_id {self.inputs_by_step_id}")
             elif step_id in self.inputs_by_step_id:
                 outputs["output"] = self.inputs_by_step_id[step_id]
 

From 146f52c5fb9c205c3c1200cb6941a8808c67cddc Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 19 Sep 2022 17:18:52 +0200
Subject: [PATCH 5/9] Move input_optional to Step for re-use

---
 lib/galaxy/model/__init__.py       | 8 ++++++--
 lib/galaxy/workflow/run_request.py | 2 +-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 0fff633aede..248be424131 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -7063,13 +7063,17 @@ class WorkflowStep(Base, RepresentById):
 
     @property
     def input_default_value(self):
-        tool_inputs = self.tool_inputs
-        tool_state = tool_inputs
+        tool_state = self.tool_inputs
         default_value = tool_state.get("default")
         if default_value:
             default_value = json.loads(default_value)["value"]
         return default_value
 
+    @property
+    def input_optional(self):
+        tool_state = self.tool_inputs
+        return tool_state.get("optional") or False
+
     def get_input(self, input_name):
         for step_input in self.inputs:
             if step_input.name == input_name:
diff --git a/lib/galaxy/workflow/run_request.py b/lib/galaxy/workflow/run_request.py
index 6cb858ac948..e808f1edabc 100644
--- a/lib/galaxy/workflow/run_request.py
+++ b/lib/galaxy/workflow/run_request.py
@@ -88,7 +88,7 @@ def _normalize_inputs(steps, inputs, inputs_by):
             if possible_input_key in inputs:
                 inputs_key = possible_input_key
         default_value = step.tool_inputs.get("default")
-        optional = step.tool_inputs.get("optional") or False
+        optional = step.input_optional
         # Need to be careful here to make sure 'default' has correct type - not sure how to do that
         # but asserting 'optional' is definitely a bool and not a String->Bool or something is a good
         # start to ensure tool state is being preserved and loaded in a type safe way.

From 58142b8f44361aefbcd1ea0ad02b6ca71b36eedc Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Tue, 20 Sep 2022 13:31:01 +0200
Subject: [PATCH 6/9] Set output for optional inputs in set_outputs_for_input

This is the core of the backend fix and allows omitting optional
workflow inputs from the API. This somewhat worked previously when
`recover_mapping` didin't have to be called, but failed when a step
had to be delayed.
---
 lib/galaxy/workflow/run.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/workflow/run.py b/lib/galaxy/workflow/run.py
index 5815ee0438a..3a0fcb0036c 100644
--- a/lib/galaxy/workflow/run.py
+++ b/lib/galaxy/workflow/run.py
@@ -388,7 +388,7 @@ class WorkflowProgress:
             step_id = step.id
             if step_id not in self.inputs_by_step_id and "output" not in outputs:
                 default_value = step.input_default_value
-                if default_value:
+                if default_value or step.input_optional:
                     outputs["output"] = default_value
                 else:
                     raise ValueError(f"{step.log_str()} not found in inputs_step_id {self.inputs_by_step_id}")

From 64cb245ddd508288a2113b73ca7570cd0479c168 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 19 Sep 2022 17:46:04 +0200
Subject: [PATCH 7/9] Fix running subworkflows with disconnected
 optional/default inputs

---
 lib/galaxy/model/__init__.py                  | 4 ++++
 lib/galaxy/workflow/run.py                    | 6 +++---
 test/unit/workflows/test_workflow_progress.py | 2 ++
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 248be424131..2b711b4d42a 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -7050,6 +7050,10 @@ class WorkflowStep(Base, RepresentById):
         self.uuid = uuid4()
         self._input_connections_by_name = None
 
+    @reconstructor
+    def init_on_load(self):
+        self._input_connections_by_name = None
+
     @property
     def tool_uuid(self):
         return self.dynamic_tool and self.dynamic_tool.uuid
diff --git a/lib/galaxy/workflow/run.py b/lib/galaxy/workflow/run.py
index 3a0fcb0036c..fd5ede8fb8d 100644
--- a/lib/galaxy/workflow/run.py
+++ b/lib/galaxy/workflow/run.py
@@ -466,9 +466,9 @@ class WorkflowProgress:
         subworkflow_inputs = {}
         for input_subworkflow_step in subworkflow.input_steps:
             connection_found = False
+            subworkflow_step_id = input_subworkflow_step.id
             for input_connection in step.input_connections:
-                if input_connection.input_subworkflow_step == input_subworkflow_step:
-                    subworkflow_step_id = input_subworkflow_step.id
+                if input_connection.input_subworkflow_step_id == subworkflow_step_id:
                     is_data = input_connection.output_step.type != "parameter_input"
                     replacement = self.replacement_for_connection(
                         input_connection,
@@ -478,7 +478,7 @@ class WorkflowProgress:
                     connection_found = True
                     break
 
-            if not connection_found:
+            if not connection_found and not input_subworkflow_step.input_optional:
                 raise Exception("Could not find connections for all subworkflow inputs.")
 
         return WorkflowProgress(subworkflow_invocation, subworkflow_inputs, self.module_injector, param_map=param_map)
diff --git a/test/unit/workflows/test_workflow_progress.py b/test/unit/workflows/test_workflow_progress.py
index 3b173c886cf..32eb90f6f6c 100644
--- a/test/unit/workflows/test_workflow_progress.py
+++ b/test/unit/workflows/test_workflow_progress.py
@@ -189,6 +189,8 @@ class WorkflowProgressTestCase(unittest.TestCase):
         subworkflow_invocation = self.invocation.create_subworkflow_invocation_for_step(
             self.invocation.workflow.step_by_index(1)
         )
+        self.app.model.session.add(subworkflow_invocation)
+        self.app.model.session.flush()
         progress = self._new_workflow_progress()
         remaining_steps = progress.remaining_steps()
         (subworkflow_step, subworkflow_invocation_step) = remaining_steps[0]

From 9a7c23335fc79d65648d033624989f4d776754fb Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Tue, 20 Sep 2022 11:47:01 +0200
Subject: [PATCH 8/9] Regression test for unspecified optional data inputs

---
 lib/galaxy/dependencies/dev-requirements.txt  |  2 +-
 .../dependencies/pinned-requirements.txt      |  2 +-
 lib/galaxy_test/api/test_workflows.py         | 44 +++++++++++++++++++
 3 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt
index 917865e7188..1fc4613b3c7 100644
--- a/lib/galaxy/dependencies/dev-requirements.txt
+++ b/lib/galaxy/dependencies/dev-requirements.txt
@@ -72,7 +72,7 @@ galaxy-sequence-utils==1.1.5
 gravity==0.13.3; python_version >= "3.6"
 greenlet==1.1.2; python_version >= "3" and python_full_version < "3.0.0" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0") or python_version >= "3" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0") and python_full_version >= "3.5.0"
 gunicorn==20.1.0; python_version >= "3.5"
-gxformat2==0.15.0
+gxformat2==0.16.0
 h11==0.12.0; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.7.0"
 h5py==3.7.0; python_version >= "3.7"
 httpcore==0.15.0; python_version >= "3.7"
diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt
index ab447226945..b76a931ef47 100644
--- a/lib/galaxy/dependencies/pinned-requirements.txt
+++ b/lib/galaxy/dependencies/pinned-requirements.txt
@@ -60,7 +60,7 @@ galaxy-sequence-utils==1.1.5
 gravity==0.13.3; python_version >= "3.6"
 greenlet==1.1.2; python_version >= "3" and python_full_version < "3.0.0" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0") or python_version >= "3" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0") and python_full_version >= "3.5.0"
 gunicorn==20.1.0; python_version >= "3.5"
-gxformat2==0.15.0
+gxformat2==0.16.0
 h11==0.12.0; python_version >= "3.7"
 h5py==3.7.0; python_version >= "3.7"
 humanfriendly==10.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0"
diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py
index 92045bcad69..5658e5b9155 100644
--- a/lib/galaxy_test/api/test_workflows.py
+++ b/lib/galaxy_test/api/test_workflows.py
@@ -3172,6 +3172,50 @@ input1:
             content = self.dataset_populator.get_history_dataset_content(history_id)
             assert "No input selected" in content
 
+    def test_run_with_optional_data_unspecified_survives_delayed_step(self):
+        with self.dataset_populator.test_history() as history_id:
+            self._run_workflow(
+                """
+class: GalaxyWorkflow
+inputs:
+  required:
+    type: data
+  optional:
+    type: data
+    optional: true
+outputs:
+  out1:
+    outputSource: count_multi_file/out_file1
+steps:
+  expression:
+    tool_id: expression_parse_int
+    state:
+      input1: 1
+  head:
+    tool_id: head
+    in:
+      input: required
+    state:
+      lineNum:
+        $link:  expression/out1
+  count_multi_file:
+    tool_id: count_multi_file
+    in:
+      input1:
+      - optional
+      - head/out_file1
+    out:
+      out_file1: out_file1
+test_data:
+  required:
+    value: 1.bed
+    type: File
+""",
+                history_id=history_id,
+                wait=True,
+                assert_ok=True,
+            )
+
     def test_run_with_non_optional_data_unspecified_fails_invocation(self):
         with self.dataset_populator.test_history() as history_id:
             error = self._run_jobs(

From 17f3647a6f9ef24d7928aedb862565e852a60252 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Tue, 20 Sep 2022 12:18:54 +0200
Subject: [PATCH 9/9] Test that subworkflows with disconnected optional inputs
 are scheduled correctly

---
 lib/galaxy_test/api/test_workflows.py     | 46 ++++++++++-------------
 lib/galaxy_test/base/populators.py        |  5 ++-
 lib/galaxy_test/base/workflow_fixtures.py | 38 +++++++++++++++++++
 3 files changed, 61 insertions(+), 28 deletions(-)

diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py
index 5658e5b9155..1b318e69e5e 100644
--- a/lib/galaxy_test/api/test_workflows.py
+++ b/lib/galaxy_test/api/test_workflows.py
@@ -16,6 +16,7 @@ from typing import (
 from uuid import uuid4
 
 import pytest
+import yaml
 from requests import (
     delete,
     get,
@@ -41,6 +42,7 @@ from galaxy_test.base.workflow_fixtures import (
     WORKFLOW_ONE_STEP_DEFAULT,
     WORKFLOW_OPTIONAL_FALSE_INPUT_COLLECTION,
     WORKFLOW_OPTIONAL_FALSE_INPUT_DATA,
+    WORKFLOW_OPTIONAL_INPUT_DELAYED_SCHEDULING,
     WORKFLOW_OPTIONAL_TRUE_INPUT_COLLECTION,
     WORKFLOW_OPTIONAL_TRUE_INPUT_DATA,
     WORKFLOW_PARAMETER_INPUT_INTEGER_DEFAULT,
@@ -3175,42 +3177,32 @@ input1:
     def test_run_with_optional_data_unspecified_survives_delayed_step(self):
         with self.dataset_populator.test_history() as history_id:
             self._run_workflow(
+                WORKFLOW_OPTIONAL_INPUT_DELAYED_SCHEDULING,
+                history_id=history_id,
+                wait=True,
+                assert_ok=True,
+            )
+
+    def test_run_subworkflow_with_optional_data_unspecified(self):
+        with self.dataset_populator.test_history() as history_id:
+            subworkflow = yaml.safe_load(
                 """
 class: GalaxyWorkflow
 inputs:
-  required:
-    type: data
-  optional:
-    type: data
-    optional: true
-outputs:
-  out1:
-    outputSource: count_multi_file/out_file1
+  required: data
 steps:
-  expression:
-    tool_id: expression_parse_int
-    state:
-      input1: 1
-  head:
-    tool_id: head
+  nested_workflow:
     in:
-      input: required
-    state:
-      lineNum:
-        $link:  expression/out1
-  count_multi_file:
-    tool_id: count_multi_file
-    in:
-      input1:
-      - optional
-      - head/out_file1
-    out:
-      out_file1: out_file1
+      required: required
 test_data:
   required:
     value: 1.bed
     type: File
-""",
+"""
+            )
+            subworkflow["steps"]["nested_workflow"]["run"] = yaml.safe_load(WORKFLOW_OPTIONAL_INPUT_DELAYED_SCHEDULING)
+            self._run_workflow(
+                subworkflow,
                 history_id=history_id,
                 wait=True,
                 assert_ok=True,
diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py
index a17ea32e4dc..4c9765dc2f1 100644
--- a/lib/galaxy_test/base/populators.py
+++ b/lib/galaxy_test/base/populators.py
@@ -1656,7 +1656,10 @@ class BaseWorkflowPopulator(BasePopulator):
         if test_data is None:
             if jobs_descriptions is None:
                 assert source_type != "path"
-                jobs_descriptions = yaml.safe_load(has_workflow)
+                if isinstance(has_workflow, dict):
+                    jobs_descriptions = has_workflow
+                else:
+                    jobs_descriptions = yaml.safe_load(has_workflow)
 
             test_data = jobs_descriptions.get("test_data", {})
 
diff --git a/lib/galaxy_test/base/workflow_fixtures.py b/lib/galaxy_test/base/workflow_fixtures.py
index bd0632633ed..f01a3c7a13e 100644
--- a/lib/galaxy_test/base/workflow_fixtures.py
+++ b/lib/galaxy_test/base/workflow_fixtures.py
@@ -614,6 +614,44 @@ steps:
 """
 
 
+WORKFLOW_OPTIONAL_INPUT_DELAYED_SCHEDULING = """
+class: GalaxyWorkflow
+inputs:
+  required:
+    type: data
+  optional:
+    type: data
+    optional: true
+outputs:
+  out1:
+    outputSource: count_multi_file/out_file1
+steps:
+  expression:
+    tool_id: expression_parse_int
+    state:
+      input1: 1
+  head:
+    tool_id: head
+    in:
+      input: required
+    state:
+      lineNum:
+        $link:  expression/out1
+  count_multi_file:
+    tool_id: count_multi_file
+    in:
+      input1:
+      - optional
+      - head/out_file1
+    out:
+      out_file1: out_file1
+test_data:
+  required:
+    value: 1.bed
+    type: File
+"""
+
+
 WORKFLOW_RUNTIME_PARAMETER_SIMPLE = """
 class: GalaxyWorkflow
 inputs: