diff --git a/lib/galaxy/managers/workflows.py b/lib/galaxy/managers/workflows.py index 2e827d23256..945fc2d3235 100644 --- a/lib/galaxy/managers/workflows.py +++ b/lib/galaxy/managers/workflows.py @@ -525,11 +525,17 @@ class WorkflowContentsManager(UsesAnnotations): workflow.license = data.get("license") workflow.creator_metadata = data.get("creator") - if "license" in data: - workflow.license = data["license"] - - if "creator" in data: - workflow.creator_metadata = data["creator"] + if hasattr(workflow_state_resolution_options, "archive_source"): + if workflow_state_resolution_options.archive_source: + source_metadata = {} + if workflow_state_resolution_options.archive_source == "trs_tool": + source_metadata["trs_tool_id"] = workflow_state_resolution_options.trs_tool_id + source_metadata["trs_version_id"] = workflow_state_resolution_options.trs_version_id + source_metadata["trs_server"] = workflow_state_resolution_options.trs_server + elif not workflow_state_resolution_options.archive_source.startswith("file://"): # URL import + source_metadata["url"] = workflow_state_resolution_options.archive_source + workflow_state_resolution_options.archive_source = None # so trs_id is not set for subworkflows + workflow.source_metadata = source_metadata # Assume no errors until we find a step that has some workflow.has_errors = False @@ -855,6 +861,7 @@ class WorkflowContentsManager(UsesAnnotations): data["report"] = workflow.reports_config or {} data["license"] = workflow.license data["creator"] = workflow.creator_metadata + data["source_metadata"] = workflow.source_metadata data["annotation"] = self.get_item_annotation_str(trans.sa_session, trans.user, stored) or "" output_label_index = set() @@ -1094,6 +1101,8 @@ class WorkflowContentsManager(UsesAnnotations): data["creator"] = workflow.creator_metadata if workflow.license: data["license"] = workflow.license + if workflow.source_metadata: + data["source_metadata"] = workflow.source_metadata # For each step, rebuild the form and encode the state for step in workflow.steps: # Load from database representation @@ -1644,6 +1653,12 @@ class WorkflowCreateOptions(WorkflowStateResolutionOptions): tool_panel_section_mapping: Dict = {} shed_tool_conf: Optional[str] = None + # for workflows imported by archive source + archive_source: str = "" + trs_tool_id: str = "" + trs_version_id: str = "" + trs_server: str = "" + @property def is_importable(self): # if self.importable is None, use self.publish that has a default. diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 2be42fc989c..e133999e0e1 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6635,9 +6635,10 @@ class Workflow(Base, Dictifiable, RepresentById): name = Column(TEXT) has_cycles = Column(Boolean) has_errors = Column(Boolean) - reports_config = Column(MutableJSONType) - creator_metadata = Column(MutableJSONType) + reports_config = Column(JSONType) + creator_metadata = Column(JSONType) license = Column(TEXT) + source_metadata = Column(JSONType) uuid = Column(UUIDType, nullable=True) steps = relationship( diff --git a/lib/galaxy/model/migrate/versions/0181_add_source_metadata_workflow_table.py b/lib/galaxy/model/migrate/versions/0181_add_source_metadata_workflow_table.py new file mode 100644 index 00000000000..59755972891 --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0181_add_source_metadata_workflow_table.py @@ -0,0 +1,35 @@ +""" +Migration script for adding source_metadata column to workflow table. +""" + +import logging + +from sqlalchemy import ( + Column, + MetaData, +) + +from galaxy.model.custom_types import JSONType +from galaxy.model.migrate.versions.util import ( + add_column, + drop_column, +) + +log = logging.getLogger(__name__) +metadata = MetaData() + + +def upgrade(migrate_engine): + print(__doc__) + metadata.bind = migrate_engine + metadata.reflect() + + source_metadata_column = Column("source_metadata", JSONType) + add_column(source_metadata_column, "workflow", metadata) + + +def downgrade(migrate_engine): + metadata.bind = migrate_engine + metadata.reflect() + + drop_column("source_metadata", "workflow", metadata) diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index 42a8687de52..8ba9298a2f3 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -645,6 +645,41 @@ steps: other_import_response = self.__import_workflow(workflow_id) self._assert_status_code_is(other_import_response, 403) + def test_url_import(self): + url = "https://raw.githubusercontent.com/galaxyproject/galaxy/release_19.09/test/base/data/test_workflow_1.ga" + workflow_id = self._post("workflows", data={"archive_source": url}).json()["id"] + workflow = self._download_workflow(workflow_id) + assert "TestWorkflow1" in workflow["name"] + assert ( + workflow.get("source_metadata").get("url") == url + ) # disappearance of source_metadata on modification is tested in test_trs_import + + def test_trs_import(self): + trs_payload = { + "archive_source": "trs_tool", + "trs_server": "dockstore", + "trs_tool_id": "#workflow/github.com/jmchilton/galaxy-workflow-dockstore-example-1/mycoolworkflow", + "trs_version_id": "master", + } + workflow_id = self._post("workflows", data=trs_payload).json()["id"] + original_workflow = self._download_workflow(workflow_id) + assert "Test Workflow" in original_workflow["name"] + assert original_workflow.get("source_metadata").get("trs_tool_id") == trs_payload["trs_tool_id"] + assert original_workflow.get("source_metadata").get("trs_version_id") == trs_payload["trs_version_id"] + + # refactor workflow and check that the trs id is removed + actions = [ + {"action_type": "update_step_label", "step": {"order_index": 0}, "label": "new_label"}, + ] + self.workflow_populator.refactor_workflow(workflow_id, actions) + refactored_workflow = self._download_workflow(workflow_id) + assert refactored_workflow.get("source_metadata") is None + + # reupload original_workflow and check that the trs id is removed + reuploaded_workflow_id = self.workflow_populator.create_workflow(original_workflow) + reuploaded_workflow = self._download_workflow(reuploaded_workflow_id) + assert reuploaded_workflow.get("source_metadata") is None + def test_anonymous_published(self): def anonymous_published_workflows(): workflows_url = self._api_url("workflows?show_published=True") diff --git a/test/unit/data/model/mapping/test_model_mapping.py b/test/unit/data/model/mapping/test_model_mapping.py index 924e03eff6f..93e16eb9b44 100644 --- a/test/unit/data/model/mapping/test_model_mapping.py +++ b/test/unit/data/model/mapping/test_model_mapping.py @@ -5352,6 +5352,7 @@ class TestWorkflow(BaseTest): reports_config = "b" creator_metadata = "c" license = "d" + source_metadata = "e" uuid = uuid4() obj = cls_() @@ -5365,6 +5366,7 @@ class TestWorkflow(BaseTest): obj.reports_config = reports_config obj.creator_metadata = creator_metadata obj.license = license + obj.source_metadata = source_metadata obj.uuid = uuid with dbcleanup(session, obj) as obj_id: @@ -5380,6 +5382,7 @@ class TestWorkflow(BaseTest): assert stored_obj.reports_config == reports_config assert stored_obj.creator_metadata == creator_metadata assert stored_obj.license == license + assert stored_obj.source_metadata == source_metadata assert stored_obj.uuid == uuid def test_relationships(self, session, cls_, stored_workflow, workflow, workflow_step_factory): diff --git a/test/unit/data/test_mutable_json_column.py b/test/unit/data/test_mutable_json_column.py index 95a36ec7934..81788af4bda 100644 --- a/test/unit/data/test_mutable_json_column.py +++ b/test/unit/data/test_mutable_json_column.py @@ -9,64 +9,64 @@ class MutableColumnTest(BaseModelTestCase): item_id = item.id self.model.session.flush() self.model.session.expunge_all() - return self.model.session.query(model.Workflow).get(item_id) + return self.model.session.query(model.DynamicTool).get(item_id) def test_metadata_mutable_column(self): - w = model.Workflow() + w = model.DynamicTool() self.model.session.add(w) self.model.session.flush() - w.reports_config = {"x": "z"} + w.value = {"x": "z"} persisted = self.persist_and_reload(w) - assert persisted.reports_config == {"x": "z"} - persisted.reports_config["x"] = "1" + assert persisted.value == {"x": "z"} + persisted.value["x"] = "1" persisted = self.persist_and_reload(persisted) - assert persisted.reports_config["x"] == "1" + assert persisted.value["x"] == "1" # test string - persisted.reports_config = "abcdefg" + persisted.value = "abcdefg" persisted = self.persist_and_reload(persisted) - assert persisted.reports_config == "abcdefg" + assert persisted.value == "abcdefg" # test int - persisted.reports_config = 1 + persisted.value = 1 persisted = self.persist_and_reload(persisted) - assert persisted.reports_config == 1 + assert persisted.value == 1 # test float - persisted.reports_config = 1.1 + persisted.value = 1.1 persisted = self.persist_and_reload(persisted) - assert persisted.reports_config == 1.1 + assert persisted.value == 1.1 # test bool - persisted.reports_config = True + persisted.value = True persisted = self.persist_and_reload(persisted) - assert persisted.reports_config is True + assert persisted.value is True # Test nested dict/list - persisted.reports_config = {"list": [[1, 2, 3]]} + persisted.value = {"list": [[1, 2, 3]]} persisted = self.persist_and_reload(persisted) - assert persisted.reports_config == {"list": [[1, 2, 3]]} - copy.deepcopy(persisted.reports_config) - assert persisted.reports_config.pop("list") == [[1, 2, 3]] + assert persisted.value == {"list": [[1, 2, 3]]} + copy.deepcopy(persisted.value) + assert persisted.value.pop("list") == [[1, 2, 3]] persisted = self.persist_and_reload(persisted) - assert persisted.reports_config == {} - persisted.reports_config.update({"x": "z"}) + assert persisted.value == {} + persisted.value.update({"x": "z"}) persisted = self.persist_and_reload(persisted) - assert persisted.reports_config == {"x": "z"} - del persisted.reports_config["x"] + assert persisted.value == {"x": "z"} + del persisted.value["x"] persisted = self.persist_and_reload(persisted) - assert persisted.reports_config == {} - persisted.reports_config = {"x": {"y": "z"}} + assert persisted.value == {} + persisted.value = {"x": {"y": "z"}} persisted = self.persist_and_reload(persisted) - assert persisted.reports_config == {"x": {"y": "z"}} + assert persisted.value == {"x": {"y": "z"}} # These tests are failing ... at least since 20.09, # but nested mutable change tracking might have # never worked - # persisted.reports_config['x']['y'] = 'x' + # persisted.value['x']['y'] = 'x' # persisted = self.persist_and_reload(persisted) - # assert persisted.reports_config == {'x': {'y': 'x'}} - # persisted.reports_config[0].append(2) + # assert persisted.value == {'x': {'y': 'x'}} + # persisted.value[0].append(2) # persisted = self.persist_and_reload(persisted) - # assert persisted.reports_config[0] == [1, 2] - # persisted.reports_config[0].extend([3, 4]) + # assert persisted.value[0] == [1, 2] + # persisted.value[0].extend([3, 4]) # persisted = self.persist_and_reload(persisted) - # assert persisted.reports_config[0] == [1, 2, 3, 4] - # persisted.reports_config[0].remove(4) + # assert persisted.value[0] == [1, 2, 3, 4] + # persisted.value[0].remove(4) # persisted = self.persist_and_reload(persisted) - # assert persisted.reports_config[0] == [1, 2, 3] + # assert persisted.value[0] == [1, 2, 3]