mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
John Chilton
67efd072ae
We don't track workflow step inputs in any formal way in our model currently. This has resulted in some current hacks and prevents future enhancements. This commit splits WorkflowStepConnection into two models WorkflowStepInput and WorkflowStepConnection - normalizing the previous table workflow_step_connection on input step and input name. In terms of current hacks forced on it by restricting all of tool state to be confined to a big JSON blob in the database - we have problems distinguishing keys and values when walking tool state. As we store more and more JSON blobs inside of the giant tool state blob - the worse this problem gets. Take for instance checking for runtime parameters or the rules parameter values - these both use JSON blobs that aren't simple values, so it is hard to tell looking at the tool state blob in the database or the workflow export to tell what is a key or what is a value. Tracking state as normalized inputs with default values and explicit attributes runtime values should allow much more percise state definition and construction. This variant of the models would also potentially allow defining runtime values with non-tool default values (so default values defined for the workflow but still explicitly settable at runtime). The combinations of overriding defaults and defining runtime values were not representable before. In terms of future enhancements, there is a lot we cannot track with the current models - such as map/reduce options for collection operations (https://github.com/galaxyproject/galaxy/issues/4623#issuecomment-389544980). This should enable a lot of that. Obviously there are a lot of attributes defined here that are not yet utilized, but I'm using most (all?) of them downstream in the CWL branch. I'd rather populate this table fully realized and fill in the implementation around it as work continues to stream in from the CWL branch - to keep things simple and avoid extra database migrations. But I understand if this feels like speculative complexity we want to avoid despite the implementation being readily available for inspection downstream.
298 lines
8.8 KiB
Python
298 lines
8.8 KiB
Python
import json
|
|
|
|
import mock
|
|
|
|
from galaxy import model
|
|
from galaxy.util import bunch
|
|
from galaxy.workflow import modules
|
|
from .workflow_support import MockTrans, yaml_to_model
|
|
|
|
|
|
def test_input_has_no_errors():
|
|
trans = MockTrans()
|
|
input_step_module = modules.module_factory.from_dict(trans, {"type": "data_input"})
|
|
assert not input_step_module.get_errors()
|
|
|
|
|
|
def test_valid_new_tool_has_no_errors():
|
|
trans = MockTrans()
|
|
mock_tool = __mock_tool()
|
|
trans.app.toolbox.tools["cat1"] = mock_tool
|
|
tool_module = modules.module_factory.from_dict(trans, {"type": "tool", "tool_id": "cat1"})
|
|
assert not tool_module.get_errors()
|
|
|
|
|
|
def test_data_input_default_state():
|
|
trans = MockTrans()
|
|
module = modules.module_factory.from_dict(trans, {"type": "data_input", "label": "Input Dataset"})
|
|
__assert_has_runtime_input(module, label="Input Dataset")
|
|
|
|
|
|
def test_data_input_modified_state():
|
|
module = __from_state({"type": "data_input", "label": "Cool Input"})
|
|
__assert_has_runtime_input(module, label="Cool Input")
|
|
|
|
|
|
def test_data_input_step_modified_state():
|
|
module = __from_step(type="data_input", label="Cool Input")
|
|
__assert_has_runtime_input(module, label="Cool Input")
|
|
|
|
|
|
def test_data_input_compute_runtime_state_default():
|
|
module = __from_step(type="data_input")
|
|
state, errors = module.compute_runtime_state(module.trans, module.test_step)
|
|
assert not errors
|
|
assert "input" in state.inputs
|
|
assert state.inputs["input"] is None
|
|
|
|
|
|
def test_data_input_compute_runtime_state_args():
|
|
module = __from_step(type="data_input")
|
|
tool_state = module.get_state()
|
|
hda = model.HistoryDatasetAssociation()
|
|
with mock.patch("galaxy.workflow.modules.check_param") as check_method:
|
|
check_method.return_value = (hda, None)
|
|
state, errors = module.compute_runtime_state(module.trans, module.test_step, {"input": 4, "tool_state": tool_state})
|
|
assert not errors
|
|
assert "input" in state.inputs
|
|
assert state.inputs["input"] is hda
|
|
|
|
|
|
def test_data_input_connections():
|
|
module = __from_step(type="data_input")
|
|
assert len(module.get_data_inputs()) == 0
|
|
outputs = module.get_data_outputs()
|
|
assert len(outputs) == 1
|
|
output = outputs[0]
|
|
assert output["name"] == "output"
|
|
assert output["extensions"] == ["input"]
|
|
|
|
|
|
def test_data_collection_input_default_state():
|
|
trans = MockTrans()
|
|
module = modules.module_factory.from_dict(trans, {"type": "data_collection_input", "label": "Input Dataset Collection"})
|
|
__assert_has_runtime_input(module, label="Input Dataset Collection", collection_type="list")
|
|
|
|
|
|
def test_data_input_collection_modified_state():
|
|
module = __from_state({
|
|
"type": "data_collection_input",
|
|
"label": "Cool Input Collection",
|
|
"tool_state": json.dumps({"collection_type": "list:paired"}),
|
|
})
|
|
__assert_has_runtime_input(module, label="Cool Input Collection", collection_type="list:paired")
|
|
|
|
|
|
def test_data_input_collection_step_modified_state():
|
|
module = __from_step(
|
|
type="data_collection_input",
|
|
label="Cool Input Collection",
|
|
tool_inputs={
|
|
"collection_type": "list:paired",
|
|
},
|
|
)
|
|
__assert_has_runtime_input(module, label="Cool Input Collection", collection_type="list:paired")
|
|
|
|
|
|
def test_data_collection_input_connections():
|
|
module = __from_step(
|
|
type="data_collection_input",
|
|
tool_inputs={
|
|
"collection_type": "list:paired"
|
|
}
|
|
)
|
|
assert len(module.get_data_inputs()) == 0
|
|
outputs = module.get_data_outputs()
|
|
assert len(outputs) == 1
|
|
output = outputs[0]
|
|
assert output["name"] == "output"
|
|
assert output["extensions"] == ["input_collection"]
|
|
assert output["collection_type"] == "list:paired"
|
|
|
|
|
|
def test_data_collection_input_config_form():
|
|
module = __from_step(
|
|
type="data_collection_input",
|
|
tool_inputs={
|
|
"collection_type": "list:paired",
|
|
}
|
|
)
|
|
result = module.get_config_form()
|
|
assert result["inputs"][0]["value"], "list:paired"
|
|
|
|
|
|
def test_cannot_create_tool_modules_for_missing_tools():
|
|
trans = MockTrans()
|
|
module = modules.module_factory.from_dict(trans, {"type": "tool", "tool_id": "cat1"})
|
|
assert not module.tool
|
|
|
|
|
|
def test_updated_tool_version():
|
|
trans = MockTrans()
|
|
mock_tool = __mock_tool(id="cat1", version="0.9")
|
|
trans.app.toolbox.tools["cat1"] = mock_tool
|
|
module = __from_step(
|
|
trans=trans,
|
|
type="tool",
|
|
tool_id="cat1",
|
|
tool_version="0.7",
|
|
config=None,
|
|
)
|
|
# Make sure there is a warnin with tool id, old version,
|
|
# and new version.
|
|
for val in "cat1", "0.7", "0.9":
|
|
assert val in module.version_changes[0]
|
|
|
|
|
|
def test_tool_version_same():
|
|
trans = MockTrans()
|
|
mock_tool = __mock_tool(id="cat1", version="1.0")
|
|
trans.app.toolbox.tools["cat1"] = mock_tool
|
|
module = __from_step(
|
|
trans=trans,
|
|
type="tool",
|
|
tool_id="cat1",
|
|
tool_version="1.0",
|
|
config=None,
|
|
)
|
|
assert not module.version_changes
|
|
|
|
|
|
TEST_WORKFLOW_YAML = """
|
|
steps:
|
|
- type: "data_input"
|
|
label: "input1"
|
|
- type: "data_collection_input"
|
|
label: "input2"
|
|
- type: "tool"
|
|
tool_id: "cat1"
|
|
inputs:
|
|
input1:
|
|
connections:
|
|
- "@output_step": 0
|
|
output_name: "output"
|
|
- type: "tool"
|
|
tool_id: "cat1"
|
|
inputs:
|
|
input1:
|
|
connections:
|
|
- "@output_step": 0
|
|
output_name: "output"
|
|
workflow_outputs:
|
|
- output_name: "out_file1"
|
|
label: "out1"
|
|
- type: "tool"
|
|
tool_id: "cat1"
|
|
inputs:
|
|
input1:
|
|
connections:
|
|
- "@output_step": 2
|
|
output_name: "out_file1"
|
|
workflow_outputs:
|
|
- output_name: "out_file1"
|
|
"""
|
|
|
|
|
|
def test_subworkflow_new_inputs():
|
|
subworkflow_module = __new_subworkflow_module()
|
|
inputs = subworkflow_module.get_data_inputs()
|
|
assert len(inputs) == 2, len(inputs)
|
|
input1, input2 = inputs
|
|
assert input1["input_type"] == "dataset"
|
|
assert input1["name"] == "input1"
|
|
assert input2["input_type"] == "dataset_collection"
|
|
assert input2["name"] == "input2", input2["name"]
|
|
|
|
|
|
def test_subworkflow_new_outputs():
|
|
subworkflow_module = __new_subworkflow_module()
|
|
outputs = subworkflow_module.get_data_outputs()
|
|
assert len(outputs) == 2, len(outputs)
|
|
output1, output2 = outputs
|
|
assert output1["name"] == "out1"
|
|
assert output1["label"] == "out1"
|
|
assert output1["extensions"] == ["input"]
|
|
assert output2["name"] == "4:out_file1", output2["name"]
|
|
assert output2["label"] == "4:out_file1", output2["label"]
|
|
|
|
|
|
def __new_subworkflow_module():
|
|
trans = MockTrans()
|
|
mock_tool = __mock_tool(id="cat1", version="1.0")
|
|
trans.app.toolbox.tools["cat1"] = mock_tool
|
|
workflow = yaml_to_model(TEST_WORKFLOW_YAML)
|
|
stored_workflow = trans.save_workflow(workflow)
|
|
workflow_id = trans.app.security.encode_id(stored_workflow.id)
|
|
subworkflow_module = modules.module_factory.from_dict(trans, {"type": "subworkflow", "content_id": workflow_id})
|
|
return subworkflow_module
|
|
|
|
|
|
def __assert_has_runtime_input(module, label=None, collection_type=None):
|
|
inputs = module.get_runtime_inputs()
|
|
assert len(inputs) == 1
|
|
assert "input" in inputs
|
|
input_param = inputs["input"]
|
|
if label is not None:
|
|
assert input_param.get_label() == label, input_param.get_label()
|
|
if collection_type is not None:
|
|
assert input_param.collection_types == [collection_type]
|
|
return input_param
|
|
|
|
|
|
def __from_state(state):
|
|
trans = MockTrans()
|
|
module = modules.module_factory.from_dict(trans, state)
|
|
return module
|
|
|
|
|
|
def __from_step(**kwds):
|
|
if "trans" in kwds:
|
|
trans = kwds["trans"]
|
|
del kwds["trans"]
|
|
else:
|
|
trans = MockTrans()
|
|
step = __step(
|
|
**kwds
|
|
)
|
|
injector = modules.WorkflowModuleInjector(trans)
|
|
injector.inject(step, exact_tools=False)
|
|
module = step.module
|
|
module.test_step = step
|
|
return module
|
|
|
|
|
|
def __step(**kwds):
|
|
step = model.WorkflowStep()
|
|
for key, value in kwds.items():
|
|
setattr(step, key, value)
|
|
return step
|
|
|
|
|
|
def __mock_tool(
|
|
id="cat1",
|
|
version="1.0",
|
|
):
|
|
# For now ignoring inputs, params_from_strings, and
|
|
# check_and_update_param_values since only have unit tests for version
|
|
# handling - but need to write tests for all of this longer term.
|
|
tool = bunch.Bunch(
|
|
id=id,
|
|
version=version,
|
|
name=id,
|
|
inputs={},
|
|
outputs={'out_file1': bunch.Bunch(collection=None,
|
|
format='input',
|
|
format_source=None,
|
|
change_format=[],
|
|
label=None)},
|
|
params_from_strings=mock.Mock(),
|
|
check_and_update_param_values=mock.Mock(),
|
|
to_json=_to_json
|
|
)
|
|
|
|
return tool
|
|
|
|
|
|
def _to_json(*args, **kwargs):
|
|
return "{}"
|