opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13376 from simonbray/add-trs-col

Browse Source 
Add source_metadata attribute for workflows imported using TRS ID or via URL
This commit is contained in:
Marius van den Beek
2022-03-09 15:53:39 +01:00
committed by GitHub
parent eb0272becb da88dedbf0
commit ff72765507
6 changed files with 129 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/managers/workflows.py
+20 -5
View File
@@ -525,11 +525,17 @@ class WorkflowContentsManager(UsesAnnotations):
workflow.license = data.get("license")
workflow.creator_metadata = data.get("creator")
if "license" in data:
workflow.license = data["license"]
if "creator" in data:
workflow.creator_metadata = data["creator"]
if hasattr(workflow_state_resolution_options, "archive_source"):
if workflow_state_resolution_options.archive_source:
source_metadata = {}
if workflow_state_resolution_options.archive_source == "trs_tool":
source_metadata["trs_tool_id"] = workflow_state_resolution_options.trs_tool_id
source_metadata["trs_version_id"] = workflow_state_resolution_options.trs_version_id
source_metadata["trs_server"] = workflow_state_resolution_options.trs_server
elif not workflow_state_resolution_options.archive_source.startswith("file://"): # URL import
source_metadata["url"] = workflow_state_resolution_options.archive_source
workflow_state_resolution_options.archive_source = None # so trs_id is not set for subworkflows
workflow.source_metadata = source_metadata
# Assume no errors until we find a step that has some
workflow.has_errors = False
@@ -855,6 +861,7 @@ class WorkflowContentsManager(UsesAnnotations):
data["report"] = workflow.reports_config or {}
data["license"] = workflow.license
data["creator"] = workflow.creator_metadata
data["source_metadata"] = workflow.source_metadata
data["annotation"] = self.get_item_annotation_str(trans.sa_session, trans.user, stored) or ""
output_label_index = set()
@@ -1094,6 +1101,8 @@ class WorkflowContentsManager(UsesAnnotations):
data["creator"] = workflow.creator_metadata
if workflow.license:
data["license"] = workflow.license
if workflow.source_metadata:
data["source_metadata"] = workflow.source_metadata
# For each step, rebuild the form and encode the state
for step in workflow.steps:
# Load from database representation
@@ -1644,6 +1653,12 @@ class WorkflowCreateOptions(WorkflowStateResolutionOptions):
tool_panel_section_mapping: Dict = {}
shed_tool_conf: Optional[str] = None
# for workflows imported by archive source
archive_source: str = ""
trs_tool_id: str = ""
trs_version_id: str = ""
trs_server: str = ""
@property
def is_importable(self):
# if self.importable is None, use self.publish that has a default.
lib/galaxy/model/__init__.py
+3 -2
View File
@@ -6635,9 +6635,10 @@ class Workflow(Base, Dictifiable, RepresentById):
name = Column(TEXT)
has_cycles = Column(Boolean)
has_errors = Column(Boolean)
reports_config = Column(MutableJSONType)
creator_metadata = Column(MutableJSONType)
reports_config = Column(JSONType)
creator_metadata = Column(JSONType)
license = Column(TEXT)
source_metadata = Column(JSONType)
uuid = Column(UUIDType, nullable=True)
steps = relationship(
lib/galaxy/model/migrate/versions/0181_add_source_metadata_workflow_table.py
+35
View File
@@ -0,0 +1,35 @@
"""
Migration script for adding source_metadata column to workflow table.
"""
import logging
from sqlalchemy import (
Column,
MetaData,
)
from galaxy.model.custom_types import JSONType
from galaxy.model.migrate.versions.util import (
add_column,
drop_column,
)
log = logging.getLogger(__name__)
metadata = MetaData()
def upgrade(migrate_engine):
print(__doc__)
metadata.bind = migrate_engine
metadata.reflect()
source_metadata_column = Column("source_metadata", JSONType)
add_column(source_metadata_column, "workflow", metadata)
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
drop_column("source_metadata", "workflow", metadata)
lib/galaxy_test/api/test_workflows.py
+35
View File
@@ -645,6 +645,41 @@ steps:
other_import_response = self.__import_workflow(workflow_id)
self._assert_status_code_is(other_import_response, 403)
def test_url_import(self):
url = "https://raw.githubusercontent.com/galaxyproject/galaxy/release_19.09/test/base/data/test_workflow_1.ga"
workflow_id = self._post("workflows", data={"archive_source": url}).json()["id"]
workflow = self._download_workflow(workflow_id)
assert "TestWorkflow1" in workflow["name"]
assert (
workflow.get("source_metadata").get("url") == url
) # disappearance of source_metadata on modification is tested in test_trs_import
def test_trs_import(self):
trs_payload = {
"archive_source": "trs_tool",
"trs_server": "dockstore",
"trs_tool_id": "#workflow/github.com/jmchilton/galaxy-workflow-dockstore-example-1/mycoolworkflow",
"trs_version_id": "master",
}
workflow_id = self._post("workflows", data=trs_payload).json()["id"]
original_workflow = self._download_workflow(workflow_id)
assert "Test Workflow" in original_workflow["name"]
assert original_workflow.get("source_metadata").get("trs_tool_id") == trs_payload["trs_tool_id"]
assert original_workflow.get("source_metadata").get("trs_version_id") == trs_payload["trs_version_id"]
# refactor workflow and check that the trs id is removed
actions = [
{"action_type": "update_step_label", "step": {"order_index": 0}, "label": "new_label"},
]
self.workflow_populator.refactor_workflow(workflow_id, actions)
refactored_workflow = self._download_workflow(workflow_id)
assert refactored_workflow.get("source_metadata") is None
# reupload original_workflow and check that the trs id is removed
reuploaded_workflow_id = self.workflow_populator.create_workflow(original_workflow)
reuploaded_workflow = self._download_workflow(reuploaded_workflow_id)
assert reuploaded_workflow.get("source_metadata") is None
def test_anonymous_published(self):
def anonymous_published_workflows():
workflows_url = self._api_url("workflows?show_published=True")
test/unit/data/model/mapping/test_model_mapping.py
+3
View File
@@ -5352,6 +5352,7 @@ class TestWorkflow(BaseTest):
reports_config = "b"
creator_metadata = "c"
license = "d"
source_metadata = "e"
uuid = uuid4()
obj = cls_()
@@ -5365,6 +5366,7 @@ class TestWorkflow(BaseTest):
obj.reports_config = reports_config
obj.creator_metadata = creator_metadata
obj.license = license
obj.source_metadata = source_metadata
obj.uuid = uuid
with dbcleanup(session, obj) as obj_id:
@@ -5380,6 +5382,7 @@ class TestWorkflow(BaseTest):
assert stored_obj.reports_config == reports_config
assert stored_obj.creator_metadata == creator_metadata
assert stored_obj.license == license
assert stored_obj.source_metadata == source_metadata
assert stored_obj.uuid == uuid
def test_relationships(self, session, cls_, stored_workflow, workflow, workflow_step_factory):
test/unit/data/test_mutable_json_column.py
+33 -33
View File
@@ -9,64 +9,64 @@ class MutableColumnTest(BaseModelTestCase):
item_id = item.id
self.model.session.flush()
self.model.session.expunge_all()
return self.model.session.query(model.Workflow).get(item_id)
return self.model.session.query(model.DynamicTool).get(item_id)
def test_metadata_mutable_column(self):
w = model.Workflow()
w = model.DynamicTool()
self.model.session.add(w)
self.model.session.flush()
w.reports_config = {"x": "z"}
w.value = {"x": "z"}
persisted = self.persist_and_reload(w)
assert persisted.reports_config == {"x": "z"}
persisted.reports_config["x"] = "1"
assert persisted.value == {"x": "z"}
persisted.value["x"] = "1"
persisted = self.persist_and_reload(persisted)
assert persisted.reports_config["x"] == "1"
assert persisted.value["x"] == "1"
# test string
persisted.reports_config = "abcdefg"
persisted.value = "abcdefg"
persisted = self.persist_and_reload(persisted)
assert persisted.reports_config == "abcdefg"
assert persisted.value == "abcdefg"
# test int
persisted.reports_config = 1
persisted.value = 1
persisted = self.persist_and_reload(persisted)
assert persisted.reports_config == 1
assert persisted.value == 1
# test float
persisted.reports_config = 1.1
persisted.value = 1.1
persisted = self.persist_and_reload(persisted)
assert persisted.reports_config == 1.1
assert persisted.value == 1.1
# test bool
persisted.reports_config = True
persisted.value = True
persisted = self.persist_and_reload(persisted)
assert persisted.reports_config is True
assert persisted.value is True
# Test nested dict/list
persisted.reports_config = {"list": [[1, 2, 3]]}
persisted.value = {"list": [[1, 2, 3]]}
persisted = self.persist_and_reload(persisted)
assert persisted.reports_config == {"list": [[1, 2, 3]]}
copy.deepcopy(persisted.reports_config)
assert persisted.reports_config.pop("list") == [[1, 2, 3]]
assert persisted.value == {"list": [[1, 2, 3]]}
copy.deepcopy(persisted.value)
assert persisted.value.pop("list") == [[1, 2, 3]]
persisted = self.persist_and_reload(persisted)
assert persisted.reports_config == {}
persisted.reports_config.update({"x": "z"})
assert persisted.value == {}
persisted.value.update({"x": "z"})
persisted = self.persist_and_reload(persisted)
assert persisted.reports_config == {"x": "z"}
del persisted.reports_config["x"]
assert persisted.value == {"x": "z"}
del persisted.value["x"]
persisted = self.persist_and_reload(persisted)
assert persisted.reports_config == {}
persisted.reports_config = {"x": {"y": "z"}}
assert persisted.value == {}
persisted.value = {"x": {"y": "z"}}
persisted = self.persist_and_reload(persisted)
assert persisted.reports_config == {"x": {"y": "z"}}
assert persisted.value == {"x": {"y": "z"}}
# These tests are failing ... at least since 20.09,
# but nested mutable change tracking might have
# never worked
# persisted.reports_config['x']['y'] = 'x'
# persisted.value['x']['y'] = 'x'
# persisted = self.persist_and_reload(persisted)
# assert persisted.reports_config == {'x': {'y': 'x'}}
# persisted.reports_config[0].append(2)
# assert persisted.value == {'x': {'y': 'x'}}
# persisted.value[0].append(2)
# persisted = self.persist_and_reload(persisted)
# assert persisted.reports_config[0] == [1, 2]
# persisted.reports_config[0].extend([3, 4])
# assert persisted.value[0] == [1, 2]
# persisted.value[0].extend([3, 4])
# persisted = self.persist_and_reload(persisted)
# assert persisted.reports_config[0] == [1, 2, 3, 4]
# persisted.reports_config[0].remove(4)
# assert persisted.value[0] == [1, 2, 3, 4]
# persisted.value[0].remove(4)
# persisted = self.persist_and_reload(persisted)
# assert persisted.reports_config[0] == [1, 2, 3]
# assert persisted.value[0] == [1, 2, 3]