From 6b36088fd8b18280a308d557f5b02b037685bf86 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 18:23:22 -0400 Subject: [PATCH 001/221] Drop Tag.tagged_histories relationship --- lib/galaxy/model/__init__.py | 3 +-- test/unit/model/test_mapping.py | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 82245aedfa8..698a2ab0c89 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -8103,7 +8103,6 @@ class Tag(Base, RepresentById): name = Column(TrimmedString(255)) children = relationship('Tag', back_populates='parent') parent = relationship('Tag', back_populates='children', remote_side=[id]) - tagged_histories = relationship('HistoryTagAssociation', back_populates='tag') tagged_history_dataset_associations = relationship( 'HistoryDatasetAssociationTagAssociation', back_populates='tag') tagged_library_dataset_dataset_associations = relationship( @@ -8154,7 +8153,7 @@ class HistoryTagAssociation(Base, ItemTagAssociation, RepresentById): value = Column(TrimmedString(255), index=True) user_value = Column(TrimmedString(255), index=True) history = relationship('History', back_populates='tags') - tag = relationship('Tag', back_populates='tagged_histories') + tag = relationship('Tag') user = relationship('User') diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index cb43e981251..093d2ce2271 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4647,7 +4647,6 @@ class TestTag(BaseTest): self, session, cls_, - history_tag_association, history_dataset_association_tag_association, library_dataset_dataset_association_tag_association, page_tag_association, @@ -4668,7 +4667,6 @@ class TestTag(BaseTest): assoc_object.tag = obj getattr(obj, assoc_attribute).append(assoc_object) - add_association(history_tag_association, 'tagged_histories') add_association( history_dataset_association_tag_association, 'tagged_history_dataset_associations') add_association( @@ -4688,7 +4686,6 @@ class TestTag(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.parent.id == parent_tag.id assert stored_obj.children == [child_tag] - assert stored_obj.tagged_histories == [history_tag_association] assert (stored_obj.tagged_history_dataset_associations == [history_dataset_association_tag_association]) assert (stored_obj.tagged_library_dataset_dataset_associations From 4afb61d4d0a7e16183e06d9ed724e22f12077cbf Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 18:25:23 -0400 Subject: [PATCH 002/221] Drop Tag.tagged_history_dataset_associations relationship --- lib/galaxy/model/__init__.py | 4 +--- test/unit/model/test_mapping.py | 5 ----- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 698a2ab0c89..5b37e7cbfa9 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -8103,8 +8103,6 @@ class Tag(Base, RepresentById): name = Column(TrimmedString(255)) children = relationship('Tag', back_populates='parent') parent = relationship('Tag', back_populates='children', remote_side=[id]) - tagged_history_dataset_associations = relationship( - 'HistoryDatasetAssociationTagAssociation', back_populates='tag') tagged_library_dataset_dataset_associations = relationship( 'LibraryDatasetDatasetAssociationTagAssociation', back_populates='tag') tagged_pages = relationship('PageTagAssociation', back_populates='tag') @@ -8169,7 +8167,7 @@ class HistoryDatasetAssociationTagAssociation(Base, ItemTagAssociation, Represen value = Column(TrimmedString(255), index=True) user_value = Column(TrimmedString(255), index=True) history_dataset_association = relationship('HistoryDatasetAssociation', back_populates='tags') - tag = relationship('Tag', back_populates='tagged_history_dataset_associations') + tag = relationship('Tag') user = relationship('User') diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 093d2ce2271..66716c5b8d4 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4647,7 +4647,6 @@ class TestTag(BaseTest): self, session, cls_, - history_dataset_association_tag_association, library_dataset_dataset_association_tag_association, page_tag_association, workflow_step_tag_association, @@ -4667,8 +4666,6 @@ class TestTag(BaseTest): assoc_object.tag = obj getattr(obj, assoc_attribute).append(assoc_object) - add_association( - history_dataset_association_tag_association, 'tagged_history_dataset_associations') add_association( library_dataset_dataset_association_tag_association, 'tagged_library_dataset_dataset_associations') @@ -4686,8 +4683,6 @@ class TestTag(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.parent.id == parent_tag.id assert stored_obj.children == [child_tag] - assert (stored_obj.tagged_history_dataset_associations - == [history_dataset_association_tag_association]) assert (stored_obj.tagged_library_dataset_dataset_associations == [library_dataset_dataset_association_tag_association]) assert stored_obj.tagged_pages == [page_tag_association] From 6ea9a59219617ca4bfb64c99c8c9301ef110c5cc Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 18:26:47 -0400 Subject: [PATCH 003/221] Drop Tag.tagged_library_dataset_dataset_associations relationship --- lib/galaxy/model/__init__.py | 4 +--- test/unit/model/test_mapping.py | 6 ------ 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 5b37e7cbfa9..c475474a17d 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -8103,8 +8103,6 @@ class Tag(Base, RepresentById): name = Column(TrimmedString(255)) children = relationship('Tag', back_populates='parent') parent = relationship('Tag', back_populates='children', remote_side=[id]) - tagged_library_dataset_dataset_associations = relationship( - 'LibraryDatasetDatasetAssociationTagAssociation', back_populates='tag') tagged_pages = relationship('PageTagAssociation', back_populates='tag') tagged_workflow_steps = relationship('WorkflowStepTagAssociation', back_populates='tag') tagged_stored_workflows = relationship('StoredWorkflowTagAssociation', back_populates='tag') @@ -8184,7 +8182,7 @@ class LibraryDatasetDatasetAssociationTagAssociation(Base, ItemTagAssociation, R user_value = Column(TrimmedString(255), index=True) library_dataset_dataset_association = relationship( 'LibraryDatasetDatasetAssociation', back_populates='tags') - tag = relationship('Tag', back_populates='tagged_library_dataset_dataset_associations') + tag = relationship('Tag') user = relationship('User') diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 66716c5b8d4..5ab3088efc0 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4647,7 +4647,6 @@ class TestTag(BaseTest): self, session, cls_, - library_dataset_dataset_association_tag_association, page_tag_association, workflow_step_tag_association, stored_workflow_tag_association, @@ -4666,9 +4665,6 @@ class TestTag(BaseTest): assoc_object.tag = obj getattr(obj, assoc_attribute).append(assoc_object) - add_association( - library_dataset_dataset_association_tag_association, - 'tagged_library_dataset_dataset_associations') add_association(page_tag_association, 'tagged_pages') add_association(workflow_step_tag_association, 'tagged_workflow_steps') add_association(stored_workflow_tag_association, 'tagged_stored_workflows') @@ -4683,8 +4679,6 @@ class TestTag(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.parent.id == parent_tag.id assert stored_obj.children == [child_tag] - assert (stored_obj.tagged_library_dataset_dataset_associations - == [library_dataset_dataset_association_tag_association]) assert stored_obj.tagged_pages == [page_tag_association] assert stored_obj.tagged_workflow_steps == [workflow_step_tag_association] assert stored_obj.tagged_stored_workflows == [stored_workflow_tag_association] From 83e3a10a995e64f07d057052253e01cbfbf6f6a4 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 18:28:14 -0400 Subject: [PATCH 004/221] Drop Tag.tagged_pages relationship --- lib/galaxy/model/__init__.py | 3 +-- test/unit/model/test_mapping.py | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index c475474a17d..27f8d9ac2ea 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -8103,7 +8103,6 @@ class Tag(Base, RepresentById): name = Column(TrimmedString(255)) children = relationship('Tag', back_populates='parent') parent = relationship('Tag', back_populates='children', remote_side=[id]) - tagged_pages = relationship('PageTagAssociation', back_populates='tag') tagged_workflow_steps = relationship('WorkflowStepTagAssociation', back_populates='tag') tagged_stored_workflows = relationship('StoredWorkflowTagAssociation', back_populates='tag') tagged_visualizations = relationship('VisualizationTagAssociation', back_populates='tag') @@ -8197,7 +8196,7 @@ class PageTagAssociation(Base, ItemTagAssociation, RepresentById): value = Column(TrimmedString(255), index=True) user_value = Column(TrimmedString(255), index=True) page = relationship('Page', back_populates='tags') - tag = relationship('Tag', back_populates='tagged_pages') + tag = relationship('Tag') user = relationship('User') diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 5ab3088efc0..3dbbea91263 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4647,7 +4647,6 @@ class TestTag(BaseTest): self, session, cls_, - page_tag_association, workflow_step_tag_association, stored_workflow_tag_association, visualization_tag_association, @@ -4665,7 +4664,6 @@ class TestTag(BaseTest): assoc_object.tag = obj getattr(obj, assoc_attribute).append(assoc_object) - add_association(page_tag_association, 'tagged_pages') add_association(workflow_step_tag_association, 'tagged_workflow_steps') add_association(stored_workflow_tag_association, 'tagged_stored_workflows') add_association(visualization_tag_association, 'tagged_visualizations') @@ -4679,7 +4677,6 @@ class TestTag(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.parent.id == parent_tag.id assert stored_obj.children == [child_tag] - assert stored_obj.tagged_pages == [page_tag_association] assert stored_obj.tagged_workflow_steps == [workflow_step_tag_association] assert stored_obj.tagged_stored_workflows == [stored_workflow_tag_association] assert stored_obj.tagged_visualizations == [visualization_tag_association] From 208301f57f6ab7205fdf01688f98c95f1704f8aa Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 18:29:12 -0400 Subject: [PATCH 005/221] Drop Tag.tagged_workflow_steps relationship --- lib/galaxy/model/__init__.py | 3 +-- test/unit/model/test_mapping.py | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 27f8d9ac2ea..e9c52d4985c 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -8103,7 +8103,6 @@ class Tag(Base, RepresentById): name = Column(TrimmedString(255)) children = relationship('Tag', back_populates='parent') parent = relationship('Tag', back_populates='children', remote_side=[id]) - tagged_workflow_steps = relationship('WorkflowStepTagAssociation', back_populates='tag') tagged_stored_workflows = relationship('StoredWorkflowTagAssociation', back_populates='tag') tagged_visualizations = relationship('VisualizationTagAssociation', back_populates='tag') tagged_history_dataset_collections = relationship( @@ -8211,7 +8210,7 @@ class WorkflowStepTagAssociation(Base, ItemTagAssociation, RepresentById): value = Column(TrimmedString(255), index=True) user_value = Column(TrimmedString(255), index=True) workflow_step = relationship('WorkflowStep', back_populates='tags') - tag = relationship('Tag', back_populates='tagged_workflow_steps') + tag = relationship('Tag') user = relationship('User') diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 3dbbea91263..6572e24c913 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4647,7 +4647,6 @@ class TestTag(BaseTest): self, session, cls_, - workflow_step_tag_association, stored_workflow_tag_association, visualization_tag_association, history_dataset_collection_tag_association, @@ -4664,7 +4663,6 @@ class TestTag(BaseTest): assoc_object.tag = obj getattr(obj, assoc_attribute).append(assoc_object) - add_association(workflow_step_tag_association, 'tagged_workflow_steps') add_association(stored_workflow_tag_association, 'tagged_stored_workflows') add_association(visualization_tag_association, 'tagged_visualizations') add_association( @@ -4677,7 +4675,6 @@ class TestTag(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.parent.id == parent_tag.id assert stored_obj.children == [child_tag] - assert stored_obj.tagged_workflow_steps == [workflow_step_tag_association] assert stored_obj.tagged_stored_workflows == [stored_workflow_tag_association] assert stored_obj.tagged_visualizations == [visualization_tag_association] assert (stored_obj.tagged_history_dataset_collections From 686f4932354901a32c74644ad29e91a8dd92ae44 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 18:30:05 -0400 Subject: [PATCH 006/221] Drop Tag.tagged_stored_workflows relationship --- lib/galaxy/model/__init__.py | 3 +-- test/unit/model/test_mapping.py | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index e9c52d4985c..b66b1cc52d4 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -8103,7 +8103,6 @@ class Tag(Base, RepresentById): name = Column(TrimmedString(255)) children = relationship('Tag', back_populates='parent') parent = relationship('Tag', back_populates='children', remote_side=[id]) - tagged_stored_workflows = relationship('StoredWorkflowTagAssociation', back_populates='tag') tagged_visualizations = relationship('VisualizationTagAssociation', back_populates='tag') tagged_history_dataset_collections = relationship( 'HistoryDatasetCollectionTagAssociation', back_populates='tag') @@ -8225,7 +8224,7 @@ class StoredWorkflowTagAssociation(Base, ItemTagAssociation, RepresentById): value = Column(TrimmedString(255), index=True) user_value = Column(TrimmedString(255), index=True) stored_workflow = relationship('StoredWorkflow', back_populates='tags') - tag = relationship('Tag', back_populates='tagged_stored_workflows') + tag = relationship('Tag') user = relationship('User') diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 6572e24c913..c19d933ec9f 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4647,7 +4647,6 @@ class TestTag(BaseTest): self, session, cls_, - stored_workflow_tag_association, visualization_tag_association, history_dataset_collection_tag_association, library_dataset_collection_tag_association, @@ -4663,7 +4662,6 @@ class TestTag(BaseTest): assoc_object.tag = obj getattr(obj, assoc_attribute).append(assoc_object) - add_association(stored_workflow_tag_association, 'tagged_stored_workflows') add_association(visualization_tag_association, 'tagged_visualizations') add_association( history_dataset_collection_tag_association, 'tagged_history_dataset_collections') @@ -4675,7 +4673,6 @@ class TestTag(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.parent.id == parent_tag.id assert stored_obj.children == [child_tag] - assert stored_obj.tagged_stored_workflows == [stored_workflow_tag_association] assert stored_obj.tagged_visualizations == [visualization_tag_association] assert (stored_obj.tagged_history_dataset_collections == [history_dataset_collection_tag_association]) From 1bb6337db7b5bf103bc6cada8d6b8d163492e3e3 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 18:31:04 -0400 Subject: [PATCH 007/221] Drop Tag.tagged_visualizations relationship --- lib/galaxy/model/__init__.py | 3 +-- test/unit/model/test_mapping.py | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index b66b1cc52d4..dbc472d9efd 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -8103,7 +8103,6 @@ class Tag(Base, RepresentById): name = Column(TrimmedString(255)) children = relationship('Tag', back_populates='parent') parent = relationship('Tag', back_populates='children', remote_side=[id]) - tagged_visualizations = relationship('VisualizationTagAssociation', back_populates='tag') tagged_history_dataset_collections = relationship( 'HistoryDatasetCollectionTagAssociation', back_populates='tag') tagged_library_dataset_collections = relationship( @@ -8239,7 +8238,7 @@ class VisualizationTagAssociation(Base, ItemTagAssociation, RepresentById): value = Column(TrimmedString(255), index=True) user_value = Column(TrimmedString(255), index=True) visualization = relationship('Visualization', back_populates='tags') - tag = relationship('Tag', back_populates='tagged_visualizations') + tag = relationship('Tag') user = relationship('User') diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index c19d933ec9f..ce49a5684f2 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4647,7 +4647,6 @@ class TestTag(BaseTest): self, session, cls_, - visualization_tag_association, history_dataset_collection_tag_association, library_dataset_collection_tag_association, tool_tag_association, @@ -4662,7 +4661,6 @@ class TestTag(BaseTest): assoc_object.tag = obj getattr(obj, assoc_attribute).append(assoc_object) - add_association(visualization_tag_association, 'tagged_visualizations') add_association( history_dataset_collection_tag_association, 'tagged_history_dataset_collections') add_association( @@ -4673,7 +4671,6 @@ class TestTag(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.parent.id == parent_tag.id assert stored_obj.children == [child_tag] - assert stored_obj.tagged_visualizations == [visualization_tag_association] assert (stored_obj.tagged_history_dataset_collections == [history_dataset_collection_tag_association]) assert (stored_obj.tagged_library_dataset_collections From 8e80029c2631fca74f2c7ec67a83ca42f6fc3e48 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 18:31:51 -0400 Subject: [PATCH 008/221] Drop Tag.tagged_history_dataset_collections relationship --- lib/galaxy/model/__init__.py | 4 +--- test/unit/model/test_mapping.py | 5 ----- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index dbc472d9efd..60817e53149 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -8103,8 +8103,6 @@ class Tag(Base, RepresentById): name = Column(TrimmedString(255)) children = relationship('Tag', back_populates='parent') parent = relationship('Tag', back_populates='children', remote_side=[id]) - tagged_history_dataset_collections = relationship( - 'HistoryDatasetCollectionTagAssociation', back_populates='tag') tagged_library_dataset_collections = relationship( 'LibraryDatasetCollectionTagAssociation', back_populates='tag') tagged_tools = relationship('ToolTagAssociation', back_populates='tag') @@ -8254,7 +8252,7 @@ class HistoryDatasetCollectionTagAssociation(Base, ItemTagAssociation, Represent value = Column(TrimmedString(255), index=True) user_value = Column(TrimmedString(255), index=True) dataset_collection = relationship('HistoryDatasetCollectionAssociation', back_populates='tags') - tag = relationship('Tag', back_populates='tagged_history_dataset_collections') + tag = relationship('Tag') user = relationship('User') diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index ce49a5684f2..be6d0150fb5 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4647,7 +4647,6 @@ class TestTag(BaseTest): self, session, cls_, - history_dataset_collection_tag_association, library_dataset_collection_tag_association, tool_tag_association, ): @@ -4661,8 +4660,6 @@ class TestTag(BaseTest): assoc_object.tag = obj getattr(obj, assoc_attribute).append(assoc_object) - add_association( - history_dataset_collection_tag_association, 'tagged_history_dataset_collections') add_association( library_dataset_collection_tag_association, 'tagged_library_dataset_collections') add_association(tool_tag_association, 'tagged_tools') @@ -4671,8 +4668,6 @@ class TestTag(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.parent.id == parent_tag.id assert stored_obj.children == [child_tag] - assert (stored_obj.tagged_history_dataset_collections - == [history_dataset_collection_tag_association]) assert (stored_obj.tagged_library_dataset_collections == [library_dataset_collection_tag_association]) assert stored_obj.tagged_tools == [tool_tag_association] From 9ea049d73b13d9621756909d7c209bcba9a8f911 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 18:32:36 -0400 Subject: [PATCH 009/221] Drop Tag.tagged_library_dataset_collections relationship --- lib/galaxy/model/__init__.py | 4 +--- test/unit/model/test_mapping.py | 5 ----- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 60817e53149..345e63c0cf2 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -8103,8 +8103,6 @@ class Tag(Base, RepresentById): name = Column(TrimmedString(255)) children = relationship('Tag', back_populates='parent') parent = relationship('Tag', back_populates='children', remote_side=[id]) - tagged_library_dataset_collections = relationship( - 'LibraryDatasetCollectionTagAssociation', back_populates='tag') tagged_tools = relationship('ToolTagAssociation', back_populates='tag') def __str__(self): @@ -8268,7 +8266,7 @@ class LibraryDatasetCollectionTagAssociation(Base, ItemTagAssociation, Represent value = Column(TrimmedString(255), index=True) user_value = Column(TrimmedString(255), index=True) dataset_collection = relationship('LibraryDatasetCollectionAssociation', back_populates='tags') - tag = relationship('Tag', back_populates='tagged_library_dataset_collections') + tag = relationship('Tag') user = relationship('User') diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index be6d0150fb5..7ae6443cff3 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4647,7 +4647,6 @@ class TestTag(BaseTest): self, session, cls_, - library_dataset_collection_tag_association, tool_tag_association, ): obj = cls_() @@ -4660,16 +4659,12 @@ class TestTag(BaseTest): assoc_object.tag = obj getattr(obj, assoc_attribute).append(assoc_object) - add_association( - library_dataset_collection_tag_association, 'tagged_library_dataset_collections') add_association(tool_tag_association, 'tagged_tools') with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.parent.id == parent_tag.id assert stored_obj.children == [child_tag] - assert (stored_obj.tagged_library_dataset_collections - == [library_dataset_collection_tag_association]) assert stored_obj.tagged_tools == [tool_tag_association] From cfe9e5ee3b9b732fe534e0e7a4fb7a45bb08ac1c Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 18:33:30 -0400 Subject: [PATCH 010/221] Drop Tag.tagged_tools relationship --- lib/galaxy/model/__init__.py | 3 +-- test/unit/model/test_mapping.py | 4 ---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 345e63c0cf2..e6641f9fa55 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -8103,7 +8103,6 @@ class Tag(Base, RepresentById): name = Column(TrimmedString(255)) children = relationship('Tag', back_populates='parent') parent = relationship('Tag', back_populates='children', remote_side=[id]) - tagged_tools = relationship('ToolTagAssociation', back_populates='tag') def __str__(self): return "Tag(id=%s, type=%i, parent_id=%s, name=%s)" % (self.id, self.type or -1, self.parent_id, self.name) @@ -8280,7 +8279,7 @@ class ToolTagAssociation(Base, ItemTagAssociation, RepresentById): user_tname = Column(TrimmedString(255), index=True) value = Column(TrimmedString(255), index=True) user_value = Column(TrimmedString(255), index=True) - tag = relationship('Tag', back_populates='tagged_tools') + tag = relationship('Tag') user = relationship('User') diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 7ae6443cff3..b938fccdd6a 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4647,7 +4647,6 @@ class TestTag(BaseTest): self, session, cls_, - tool_tag_association, ): obj = cls_() parent_tag = cls_() @@ -4659,13 +4658,10 @@ class TestTag(BaseTest): assoc_object.tag = obj getattr(obj, assoc_attribute).append(assoc_object) - add_association(tool_tag_association, 'tagged_tools') - with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.parent.id == parent_tag.id assert stored_obj.children == [child_tag] - assert stored_obj.tagged_tools == [tool_tag_association] class TestTask(BaseTest): From 7bc9ece93c0996100be941b4fa02cba96452deb5 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 21:27:01 -0400 Subject: [PATCH 011/221] Temporarily comment out failing test Removed relationships used only in this test. Test may be redundant, given the mapping (that is under test here) is covered in test/unit/model/test_mappping.py --- test/unit/data/test_galaxy_mapping.py | 82 +++++++++++++-------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/test/unit/data/test_galaxy_mapping.py b/test/unit/data/test_galaxy_mapping.py index a5b347c9c55..3d0f273d9f8 100644 --- a/test/unit/data/test_galaxy_mapping.py +++ b/test/unit/data/test_galaxy_mapping.py @@ -234,47 +234,47 @@ class MappingTests(BaseModelTestCase): assert new_ldda.library_dataset.expired_datasets[0] == ldda assert target_folder.item_count == 1 - def test_tags(self): - model = self.model - - my_tag = model.Tag(name="Test Tag") - u = model.User(email="tagger@example.com", password="password") - self.persist(my_tag, u) - - def tag_and_test(taggable_object, tag_association_class, backref_name): - assert len(getattr(self.query(model.Tag).filter(model.Tag.name == "Test Tag").all()[0], backref_name)) == 0 - - tag_association = tag_association_class() - tag_association.tag = my_tag - taggable_object.tags = [tag_association] - self.persist(tag_association, taggable_object) - - assert len(getattr(self.query(model.Tag).filter(model.Tag.name == "Test Tag").all()[0], backref_name)) == 1 - - sw = model.StoredWorkflow() - sw.user = u - tag_and_test(sw, model.StoredWorkflowTagAssociation, "tagged_stored_workflows") - - h = model.History(name="History for Tagging", user=u) - tag_and_test(h, model.HistoryTagAssociation, "tagged_histories") - - d1 = model.HistoryDatasetAssociation(extension="txt", history=h, create_dataset=True, sa_session=model.session) - tag_and_test(d1, model.HistoryDatasetAssociationTagAssociation, "tagged_history_dataset_associations") - - page = model.Page() - page.user = u - tag_and_test(page, model.PageTagAssociation, "tagged_pages") - - visualization = model.Visualization() - visualization.user = u - tag_and_test(visualization, model.VisualizationTagAssociation, "tagged_visualizations") - - dataset_collection = model.DatasetCollection(collection_type="paired") - history_dataset_collection = model.HistoryDatasetCollectionAssociation(collection=dataset_collection) - tag_and_test(history_dataset_collection, model.HistoryDatasetCollectionTagAssociation, "tagged_history_dataset_collections") - - library_dataset_collection = model.LibraryDatasetCollectionAssociation(collection=dataset_collection) - tag_and_test(library_dataset_collection, model.LibraryDatasetCollectionTagAssociation, "tagged_library_dataset_collections") +# def test_tags(self): +# model = self.model +# +# my_tag = model.Tag(name="Test Tag") +# u = model.User(email="tagger@example.com", password="password") +# self.persist(my_tag, u) +# +# def tag_and_test(taggable_object, tag_association_class, backref_name): +# assert len(getattr(self.query(model.Tag).filter(model.Tag.name == "Test Tag").all()[0], backref_name)) == 0 +# +# tag_association = tag_association_class() +# tag_association.tag = my_tag +# taggable_object.tags = [tag_association] +# self.persist(tag_association, taggable_object) +# +# assert len(getattr(self.query(model.Tag).filter(model.Tag.name == "Test Tag").all()[0], backref_name)) == 1 +# +# sw = model.StoredWorkflow() +# sw.user = u +# tag_and_test(sw, model.StoredWorkflowTagAssociation, "tagged_stored_workflows") +# +# h = model.History(name="History for Tagging", user=u) +# tag_and_test(h, model.HistoryTagAssociation, "tagged_histories") +# +# d1 = model.HistoryDatasetAssociation(extension="txt", history=h, create_dataset=True, sa_session=model.session) +# tag_and_test(d1, model.HistoryDatasetAssociationTagAssociation, "tagged_history_dataset_associations") +# +# page = model.Page() +# page.user = u +# tag_and_test(page, model.PageTagAssociation, "tagged_pages") +# +# visualization = model.Visualization() +# visualization.user = u +# tag_and_test(visualization, model.VisualizationTagAssociation, "tagged_visualizations") +# +# dataset_collection = model.DatasetCollection(collection_type="paired") +# history_dataset_collection = model.HistoryDatasetCollectionAssociation(collection=dataset_collection) +# tag_and_test(history_dataset_collection, model.HistoryDatasetCollectionTagAssociation, "tagged_history_dataset_collections") +# +# library_dataset_collection = model.LibraryDatasetCollectionAssociation(collection=dataset_collection) +# tag_and_test(library_dataset_collection, model.LibraryDatasetCollectionTagAssociation, "tagged_library_dataset_collections") def test_collection_get_interface(self): model = self.model From 887948076d0d585be7c49f23f5fb115fcb68d7fa Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 22:09:03 -0400 Subject: [PATCH 012/221] Drop User.pages_shared_by_others relationship --- lib/galaxy/model/__init__.py | 3 +-- test/unit/model/test_mapping.py | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index e6641f9fa55..f8ecc977b79 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -475,7 +475,6 @@ class User(Base, Dictifiable, RepresentById): galaxy_sessions = relationship('GalaxySession', back_populates='user', order_by=lambda: desc(GalaxySession.update_time)) # type: ignore - pages_shared_by_others = relationship('PageUserShareAssociation', back_populates='user') quotas = relationship('UserQuotaAssociation', back_populates='user') social_auth = relationship('UserAuthnzToken', back_populates='user') stored_workflow_menu_entries = relationship('StoredWorkflowMenuEntry', @@ -7969,7 +7968,7 @@ class PageUserShareAssociation(Base, UserShareAssociation): id = Column(Integer, primary_key=True) page_id = Column(Integer, ForeignKey("page.id"), index=True) user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) - user = relationship('User', back_populates='pages_shared_by_others') + user = relationship('User') page = relationship('Page', back_populates='users_shared_with') diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index b938fccdd6a..ba635f49e54 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4887,7 +4887,6 @@ class TestUser(BaseTest): user_group_association, history_factory, galaxy_session, - page_user_share_association, user_quota_association, user_authnz_token, user_preference, @@ -4919,7 +4918,6 @@ class TestUser(BaseTest): obj.histories.append(history1) obj.histories.append(history2) obj.galaxy_sessions.append(galaxy_session) - obj.pages_shared_by_others.append(page_user_share_association) obj.quotas.append(user_quota_association) obj.social_auth.append(user_authnz_token) @@ -4951,7 +4949,6 @@ class TestUser(BaseTest): assert are_same_entity_collections(stored_obj.histories, [history1, history2]) assert stored_obj.active_histories == [history1] assert stored_obj.galaxy_sessions == [galaxy_session] - assert stored_obj.pages_shared_by_others == [page_user_share_association] assert stored_obj.quotas == [user_quota_association] assert stored_obj.social_auth == [user_authnz_token] assert stored_obj.stored_workflow_menu_entries == [swme] From f0394355a71a03284338238914845afd9512b82b Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 22:10:24 -0400 Subject: [PATCH 013/221] Drop User.pages relationship --- lib/galaxy/model/__init__.py | 3 +-- test/unit/model/test_mapping.py | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index f8ecc977b79..114bb2de8fb 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -495,7 +495,6 @@ class User(Base, Dictifiable, RepresentById): api_keys: 'List[APIKeys]' = relationship('APIKeys', back_populates='user', order_by=lambda: desc(APIKeys.create_time)) # type: ignore - pages = relationship('Page', back_populates='user') reset_tokens = relationship('PasswordResetToken', back_populates='user') histories_shared_by_others = relationship('HistoryUserShareAssociation', back_populates='user') data_manager_histories = relationship('DataManagerHistoryAssociation', back_populates='user') @@ -7887,7 +7886,7 @@ class Page(Base, Dictifiable, RepresentById): importable = Column(Boolean, index=True, default=False) slug = Column(TEXT) published = Column(Boolean, index=True, default=False) - user = relationship('User', back_populates='pages') + user = relationship('User') revisions = relationship( 'PageRevision', cascade="all, delete-orphan", diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index ba635f49e54..55bb9b41a28 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4891,7 +4891,6 @@ class TestUser(BaseTest): user_authnz_token, user_preference, api_keys, - page, password_reset_token, history_user_share_association, data_manager_history_association, @@ -4929,7 +4928,6 @@ class TestUser(BaseTest): obj._preferences.set(user_preference) obj.api_keys.append(api_keys) - obj.pages.append(page) obj.reset_tokens.append(password_reset_token) obj.histories_shared_by_others.append(history_user_share_association) obj.data_manager_histories.append(data_manager_history_association) @@ -4954,7 +4952,6 @@ class TestUser(BaseTest): assert stored_obj.stored_workflow_menu_entries == [swme] assert user_preference in stored_obj._preferences.values() assert stored_obj.api_keys == [api_keys] - assert stored_obj.pages == [page] assert stored_obj.reset_tokens == [password_reset_token] assert stored_obj.histories_shared_by_others == [history_user_share_association] assert stored_obj.data_manager_histories == [data_manager_history_association] From 3a678e014383aa0f6e70b71b829331d948ddda59 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 22:11:58 -0400 Subject: [PATCH 014/221] Drop User.reset_tokens relationship, test fixture --- lib/galaxy/model/__init__.py | 3 +-- test/unit/model/test_mapping.py | 11 ----------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 114bb2de8fb..4fa58f1dba3 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -495,7 +495,6 @@ class User(Base, Dictifiable, RepresentById): api_keys: 'List[APIKeys]' = relationship('APIKeys', back_populates='user', order_by=lambda: desc(APIKeys.create_time)) # type: ignore - reset_tokens = relationship('PasswordResetToken', back_populates='user') histories_shared_by_others = relationship('HistoryUserShareAssociation', back_populates='user') data_manager_histories = relationship('DataManagerHistoryAssociation', back_populates='user') workflows_shared_by_others = relationship('StoredWorkflowUserShareAssociation', back_populates='user') @@ -763,7 +762,7 @@ class PasswordResetToken(Base, _HasTable): token = Column(String(32), primary_key=True, unique=True, index=True) expiration_time = Column(DateTime) user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) - user = relationship('User', back_populates='reset_tokens') + user = relationship('User') def __init__(self, user, token=None): if token: diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 55bb9b41a28..1248b3b6d9b 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4891,7 +4891,6 @@ class TestUser(BaseTest): user_authnz_token, user_preference, api_keys, - password_reset_token, history_user_share_association, data_manager_history_association, stored_workflow_user_share_association, @@ -4928,7 +4927,6 @@ class TestUser(BaseTest): obj._preferences.set(user_preference) obj.api_keys.append(api_keys) - obj.reset_tokens.append(password_reset_token) obj.histories_shared_by_others.append(history_user_share_association) obj.data_manager_histories.append(data_manager_history_association) obj.workflows_shared_by_others.append(stored_workflow_user_share_association) @@ -4952,7 +4950,6 @@ class TestUser(BaseTest): assert stored_obj.stored_workflow_menu_entries == [swme] assert user_preference in stored_obj._preferences.values() assert stored_obj.api_keys == [api_keys] - assert stored_obj.reset_tokens == [password_reset_token] assert stored_obj.histories_shared_by_others == [history_user_share_association] assert stored_obj.data_manager_histories == [data_manager_history_association] assert stored_obj.workflows_shared_by_others == [stored_workflow_user_share_association] @@ -7111,14 +7108,6 @@ def page_user_share_association(model, session): yield from dbcleanup_wrapper(session, instance) -@pytest.fixture -def password_reset_token(model, session, user): - token = get_unique_value() - instance = model.PasswordResetToken(user, token) - where_clause = type(instance).token == token - yield from dbcleanup_wrapper(session, instance, where_clause) - - @pytest.fixture def post_job_action(model, session): instance = model.PostJobAction('a') From 3e47e6019123a0f1ba5498a684252f11653d20a2 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 22:13:29 -0400 Subject: [PATCH 015/221] Drop User.histories_shared_by_others relationship --- lib/galaxy/model/__init__.py | 3 +-- test/unit/model/test_mapping.py | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 4fa58f1dba3..3ed21ee9861 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -495,7 +495,6 @@ class User(Base, Dictifiable, RepresentById): api_keys: 'List[APIKeys]' = relationship('APIKeys', back_populates='user', order_by=lambda: desc(APIKeys.create_time)) # type: ignore - histories_shared_by_others = relationship('HistoryUserShareAssociation', back_populates='user') data_manager_histories = relationship('DataManagerHistoryAssociation', back_populates='user') workflows_shared_by_others = relationship('StoredWorkflowUserShareAssociation', back_populates='user') roles = relationship('UserRoleAssociation', back_populates='user') @@ -2775,7 +2774,7 @@ class HistoryUserShareAssociation(Base, UserShareAssociation): id = Column(Integer, primary_key=True) history_id = Column(Integer, ForeignKey('history.id'), index=True) user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) - user = relationship('User', back_populates='histories_shared_by_others') + user = relationship('User') history = relationship('History', back_populates='users_shared_with') diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 1248b3b6d9b..8485fc8c666 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4891,7 +4891,6 @@ class TestUser(BaseTest): user_authnz_token, user_preference, api_keys, - history_user_share_association, data_manager_history_association, stored_workflow_user_share_association, user_role_association, @@ -4927,7 +4926,6 @@ class TestUser(BaseTest): obj._preferences.set(user_preference) obj.api_keys.append(api_keys) - obj.histories_shared_by_others.append(history_user_share_association) obj.data_manager_histories.append(data_manager_history_association) obj.workflows_shared_by_others.append(stored_workflow_user_share_association) obj.roles.append(user_role_association) @@ -4950,7 +4948,6 @@ class TestUser(BaseTest): assert stored_obj.stored_workflow_menu_entries == [swme] assert user_preference in stored_obj._preferences.values() assert stored_obj.api_keys == [api_keys] - assert stored_obj.histories_shared_by_others == [history_user_share_association] assert stored_obj.data_manager_histories == [data_manager_history_association] assert stored_obj.workflows_shared_by_others == [stored_workflow_user_share_association] assert stored_obj.roles == [user_role_association] From 0c98c15ea5a0c2bd76701c9945840da1c06757d6 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 22:15:13 -0400 Subject: [PATCH 016/221] Drop User.workflows_shared_by_others relationship --- lib/galaxy/model/__init__.py | 3 +-- test/unit/model/test_mapping.py | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 3ed21ee9861..a5b14613797 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -496,7 +496,6 @@ class User(Base, Dictifiable, RepresentById): back_populates='user', order_by=lambda: desc(APIKeys.create_time)) # type: ignore data_manager_histories = relationship('DataManagerHistoryAssociation', back_populates='user') - workflows_shared_by_others = relationship('StoredWorkflowUserShareAssociation', back_populates='user') roles = relationship('UserRoleAssociation', back_populates='user') stored_workflows = relationship('StoredWorkflow', back_populates='user', primaryjoin=(lambda: User.id == StoredWorkflow.user_id)) # type: ignore @@ -6621,7 +6620,7 @@ class StoredWorkflowUserShareAssociation(Base, UserShareAssociation): id = Column(Integer, primary_key=True) stored_workflow_id = Column(Integer, ForeignKey('stored_workflow.id'), index=True) user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) - user = relationship('User', back_populates='workflows_shared_by_others') + user = relationship('User') stored_workflow = relationship('StoredWorkflow', back_populates='users_shared_with') diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 8485fc8c666..76f67e369f6 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4892,7 +4892,6 @@ class TestUser(BaseTest): user_preference, api_keys, data_manager_history_association, - stored_workflow_user_share_association, user_role_association, stored_workflow, stored_workflow_menu_entry_factory, @@ -4927,7 +4926,6 @@ class TestUser(BaseTest): obj.api_keys.append(api_keys) obj.data_manager_histories.append(data_manager_history_association) - obj.workflows_shared_by_others.append(stored_workflow_user_share_association) obj.roles.append(user_role_association) obj.stored_workflows.append(stored_workflow) obj.visualizations_shared_by_others.append(visualization_user_share_association) @@ -4949,7 +4947,6 @@ class TestUser(BaseTest): assert user_preference in stored_obj._preferences.values() assert stored_obj.api_keys == [api_keys] assert stored_obj.data_manager_histories == [data_manager_history_association] - assert stored_obj.workflows_shared_by_others == [stored_workflow_user_share_association] assert stored_obj.roles == [user_role_association] assert stored_obj.stored_workflows == [stored_workflow] assert (stored_obj.visualizations_shared_by_others From 3237eccb00eb59f35d85cd68949099b0c80ecb33 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 22:16:20 -0400 Subject: [PATCH 017/221] Drop User.visualizations_shared_by_others --- lib/galaxy/model/__init__.py | 4 +--- test/unit/model/test_mapping.py | 4 ---- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index a5b14613797..af104188fc7 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -499,8 +499,6 @@ class User(Base, Dictifiable, RepresentById): roles = relationship('UserRoleAssociation', back_populates='user') stored_workflows = relationship('StoredWorkflow', back_populates='user', primaryjoin=(lambda: User.id == StoredWorkflow.user_id)) # type: ignore - visualizations_shared_by_others = relationship('VisualizationUserShareAssociation', - back_populates='user') preferences: association_proxy # defined at the end of this module @@ -8082,7 +8080,7 @@ class VisualizationUserShareAssociation(Base, UserShareAssociation): id = Column(Integer, primary_key=True) visualization_id = Column(Integer, ForeignKey('visualization.id'), index=True) user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) - user = relationship('User', back_populates='visualizations_shared_by_others') + user = relationship('User') visualization = relationship('Visualization', back_populates='users_shared_with') diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 76f67e369f6..c8537abc2fb 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4895,7 +4895,6 @@ class TestUser(BaseTest): user_role_association, stored_workflow, stored_workflow_menu_entry_factory, - visualization_user_share_association, ): history1 = history_factory(deleted=False) history2 = history_factory(deleted=True) @@ -4928,7 +4927,6 @@ class TestUser(BaseTest): obj.data_manager_histories.append(data_manager_history_association) obj.roles.append(user_role_association) obj.stored_workflows.append(stored_workflow) - obj.visualizations_shared_by_others.append(visualization_user_share_association) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) @@ -4949,8 +4947,6 @@ class TestUser(BaseTest): assert stored_obj.data_manager_histories == [data_manager_history_association] assert stored_obj.roles == [user_role_association] assert stored_obj.stored_workflows == [stored_workflow] - assert (stored_obj.visualizations_shared_by_others - == [visualization_user_share_association]) delete_from_database(session, [history1, history2, swme]) From d2588cd58647a3a80c20edca5ac997aa1d92f90b Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 22:21:13 -0400 Subject: [PATCH 018/221] Drop DynamicTool.workflow_steps relationship --- lib/galaxy/model/__init__.py | 4 +--- test/unit/model/test_mapping.py | 8 -------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index af104188fc7..6e4aa1dc31b 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -784,7 +784,6 @@ class DynamicTool(Base, Dictifiable, RepresentById): hidden = Column(Boolean, default=True) active = Column(Boolean, default=True) value = Column(MutableJSONType) - workflow_steps = relationship('WorkflowStep', back_populates='dynamic_tool') dict_collection_visible_keys = ('id', 'tool_id', 'tool_format', 'tool_version', 'uuid', 'active', 'hidden') dict_element_visible_keys = ('id', 'tool_id', 'tool_format', 'tool_version', 'uuid', 'active', 'hidden') @@ -6290,8 +6289,7 @@ class WorkflowStep(Base, RepresentById): primaryjoin=(lambda: Workflow.id == WorkflowStep.subworkflow_id), # type: ignore back_populates='parent_workflow_steps') dynamic_tool = relationship('DynamicTool', - primaryjoin=(lambda: DynamicTool.id == WorkflowStep.dynamic_tool_id), # type: ignore - back_populates='workflow_steps') + primaryjoin=(lambda: DynamicTool.id == WorkflowStep.dynamic_tool_id)) # type: ignore tags = relationship('WorkflowStepTagAssociation', order_by=lambda: WorkflowStepTagAssociation.id, # type: ignore back_populates='workflow_step') diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index c8537abc2fb..852c9c97008 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -894,14 +894,6 @@ class TestDynamicTool(BaseTest): assert stored_obj.active == active assert stored_obj.value == value - def test_relationships(self, session, cls_, workflow_step): - obj = cls_() - obj.workflow_steps.append(workflow_step) - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.workflow_steps == [workflow_step] - def test_construct_with_uuid(self, session, cls_): uuid = uuid4() obj = cls_(uuid=uuid) From 2383f58b823da283345e98749cec36eb440da922 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 22:26:14 -0400 Subject: [PATCH 019/221] Drop ImplicitlyCreatedDatasetCollectionInput.dataset_collection relationship --- lib/galaxy/model/__init__.py | 7 +------ test/unit/model/test_mapping.py | 8 -------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 6e4aa1dc31b..ff2fb7b7bdd 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1899,10 +1899,6 @@ class ImplicitlyCreatedDatasetCollectionInput(Base, RepresentById): primaryjoin=(lambda: HistoryDatasetCollectionAssociation.id # type: ignore == ImplicitlyCreatedDatasetCollectionInput.input_dataset_collection_id) # type: ignore ) - dataset_collection = relationship('HistoryDatasetCollectionAssociation', - primaryjoin=(lambda: HistoryDatasetCollectionAssociation.id # type: ignore - == ImplicitlyCreatedDatasetCollectionInput.dataset_collection_id), # type: ignore - back_populates='implicit_input_collections') def __init__(self, name, input_dataset_collection): self.name = name @@ -5479,8 +5475,7 @@ class HistoryDatasetCollectionAssociation( ) implicit_input_collections = relationship('ImplicitlyCreatedDatasetCollectionInput', primaryjoin=(lambda: HistoryDatasetCollectionAssociation.id # type: ignore - == ImplicitlyCreatedDatasetCollectionInput.dataset_collection_id), # type: ignore - back_populates='dataset_collection', + == ImplicitlyCreatedDatasetCollectionInput.dataset_collection_id) # type: ignore ) implicit_collection_jobs = relationship( 'ImplicitCollectionJobs', diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 852c9c97008..6ee7e4a5f61 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -2281,20 +2281,12 @@ class TestImplicitlyCreatedDatasetCollectionInput(BaseTest): session, cls_, history_dataset_collection_association, - history_dataset_collection_association_factory, ): - hdca2 = history_dataset_collection_association_factory() - persist(session, hdca2) - obj = cls_(None, history_dataset_collection_association) - obj.dataset_collection_id = hdca2.id with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.input_dataset_collection.id == history_dataset_collection_association.id - assert stored_obj.dataset_collection.id == hdca2.id - - delete_from_database(session, [hdca2]) class TestInteractiveToolEntryPoint(BaseTest): From 7fc7aa54ffb7ad1a913ab987fda6131af3b38703 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 22:29:13 -0400 Subject: [PATCH 020/221] Drop Library.info_association relationship, test fixture --- lib/galaxy/model/__init__.py | 4 +--- test/unit/model/test_mapping.py | 10 ---------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index ff2fb7b7bdd..4c384672201 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4390,7 +4390,6 @@ class Library(Base, Dictifiable, HasName, RepresentById): synopsis = Column(TEXT) root_folder = relationship('LibraryFolder', back_populates='library_root') actions = relationship('LibraryPermissions', back_populates='library') - info_association = relationship('LibraryInfoAssociation', back_populates='library') permitted_actions = get_permitted_actions(filter='LIBRARY') dict_collection_visible_keys = ['id', 'name'] @@ -4913,8 +4912,7 @@ class LibraryInfoAssociation(Base, RepresentById): lambda: and_( LibraryInfoAssociation.library_id == Library.id, # type: ignore not_(LibraryInfoAssociation.deleted)) # type: ignore - ), - back_populates='info_association') + )) template = relationship('FormDefinition', primaryjoin=lambda: LibraryInfoAssociation.form_definition_id == FormDefinition.id) # type: ignore info = relationship('FormValues', diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 6ee7e4a5f61..896429d7877 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -3094,18 +3094,14 @@ class TestLibrary(BaseTest): cls_, library_folder, library_permission, - library_info_association, ): obj = cls_(None, None, None, library_folder) obj.actions.append(library_permission) - session.add(library_info_association) # must be bound to a session for lazy load of attributes - obj.info_association.append(library_info_association) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.root_folder.id == library_folder.id assert stored_obj.actions == [library_permission] - assert stored_obj.info_association == [library_info_association] class TestLibraryDataset(BaseTest): @@ -7026,12 +7022,6 @@ def library_folder_permission(model, session, library_folder, role): yield from dbcleanup_wrapper(session, instance) -@pytest.fixture -def library_info_association(model, session, library, form_definition, form_values): - instance = model.LibraryInfoAssociation(library, form_definition, form_values) - yield from dbcleanup_wrapper(session, instance) - - @pytest.fixture def library_permission(model, session, library, role): instance = model.LibraryPermissions('a', library, role) From c6c59697b77af8fb2e367eb9998cae324a8b65a1 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 22:31:40 -0400 Subject: [PATCH 021/221] Drop DatasetColleciton.output_dataset_collections relationship --- lib/galaxy/model/__init__.py | 5 +---- test/unit/model/test_mapping.py | 4 ---- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 4c384672201..de41b667c93 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1827,8 +1827,7 @@ class JobToImplicitOutputDatasetCollectionAssociation(Base, RepresentById): job_id = Column(Integer, ForeignKey('job.id'), index=True) dataset_collection_id = Column(Integer, ForeignKey('dataset_collection.id'), index=True) name = Column(Unicode(255)) - dataset_collection = relationship( - 'DatasetCollection', back_populates="output_dataset_collections") + dataset_collection = relationship('DatasetCollection') job = relationship('Job', back_populates='output_dataset_collections') def __init__(self, name, dataset_collection): @@ -5072,8 +5071,6 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, RepresentById): primaryjoin=(lambda: DatasetCollection.id == DatasetCollectionElement.dataset_collection_id), # type: ignore back_populates='collection', order_by=lambda: DatasetCollectionElement.element_index) # type: ignore - output_dataset_collections = relationship( - 'JobToImplicitOutputDatasetCollectionAssociation', back_populates='dataset_collection') dict_collection_visible_keys = ['id', 'collection_type'] dict_element_visible_keys = ['id', 'collection_type'] diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 896429d7877..9e0f6f42b25 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -570,18 +570,14 @@ class TestDatasetCollection(BaseTest): session, cls_, dataset_collection_element, - job_to_implicit_output_dataset_collection_association, ): obj = cls_() obj.collection_type, obj.populated_state = 'a', 'b' obj.elements.append(dataset_collection_element) - obj.output_dataset_collections.append(job_to_implicit_output_dataset_collection_association) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.elements == [dataset_collection_element] - assert (stored_obj.output_dataset_collections - == [job_to_implicit_output_dataset_collection_association]) class TestDatasetCollectionElement(BaseTest): From 12eff04826a14d20e59590d1da254f07ba4d096c Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 22:33:38 -0400 Subject: [PATCH 022/221] Drop UserAuthnzToken.cloudauthz relationship --- lib/galaxy/model/__init__.py | 3 +-- test/unit/model/test_mapping.py | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index de41b667c93..400be192810 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -7693,7 +7693,6 @@ class UserAuthnzToken(Base, UserMixin, RepresentById): extra_data = Column(MutableJSONType, nullable=True) lifetime = Column(Integer) assoc_type = Column(VARCHAR(64)) - cloudauthz = relationship('CloudAuthz', back_populates='authn') user = relationship('User', back_populates='social_auth') # This static property is set at: galaxy.authnz.psa_authnz.PSAAuthnz @@ -7832,7 +7831,7 @@ class CloudAuthz(Base, _HasTable): description = Column(TEXT) create_time = Column(DateTime, default=now) user = relationship('User', back_populates='cloudauthz') - authn = relationship('UserAuthnzToken', back_populates='cloudauthz') + authn = relationship('UserAuthnzToken') def __init__(self, user_id, provider, config, authn_id, description=None): self.user_id = user_id diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 9e0f6f42b25..8f35fd89847 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -5030,13 +5030,11 @@ class TestUserAuthnzToken(BaseTest): assert stored_obj.lifetime == lifetime assert stored_obj.assoc_type == assoc_type - def test_relationships(self, session, cls_, user, cloud_authz): + def test_relationships(self, session, cls_, user): obj = cls_(get_unique_value(), None, user=user) - obj.cloudauthz.append(cloud_authz) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.cloudauthz == [cloud_authz] assert stored_obj.user.id == user.id From 38e0a645713965d6274248c57450a65821381c97 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 22:37:13 -0400 Subject: [PATCH 023/221] Drop ImpplicitCollectionJobs.history_dataset_collection_associations relationship --- lib/galaxy/model/__init__.py | 8 +------- test/unit/model/test_mapping.py | 4 ---- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 400be192810..070f6ccc111 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1911,8 +1911,6 @@ class ImplicitCollectionJobs(Base, RepresentById): populated_state = Column(TrimmedString(64), default='new', nullable=False) jobs = relationship('ImplicitCollectionJobsJobAssociation', back_populates='implicit_collection_jobs') - history_dataset_collection_associations = relationship('HistoryDatasetCollectionAssociation', - back_populates='implicit_collection_jobs') workflow_invocation_step = relationship('WorkflowInvocationStep', back_populates='implicit_collection_jobs', uselist=False) @@ -5472,11 +5470,7 @@ class HistoryDatasetCollectionAssociation( primaryjoin=(lambda: HistoryDatasetCollectionAssociation.id # type: ignore == ImplicitlyCreatedDatasetCollectionInput.dataset_collection_id) # type: ignore ) - implicit_collection_jobs = relationship( - 'ImplicitCollectionJobs', - back_populates='history_dataset_collection_associations', - uselist=False - ) + implicit_collection_jobs = relationship('ImplicitCollectionJobs', uselist=False) job = relationship( 'Job', back_populates='history_dataset_collection_associations', diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 8f35fd89847..398252c28b9 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -2137,19 +2137,15 @@ class TestImplicitCollectionJobs(BaseTest): session, cls_, implicit_collection_jobs_job_association, - history_dataset_collection_association, workflow_invocation_step, ): obj = cls_() obj.jobs.append(implicit_collection_jobs_job_association) - obj.history_dataset_collection_associations.append(history_dataset_collection_association) obj.workflow_invocation_step = workflow_invocation_step with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.jobs == [implicit_collection_jobs_job_association] - assert (stored_obj.history_dataset_collection_associations - == [history_dataset_collection_association]) assert stored_obj.workflow_invocation_step.id == workflow_invocation_step.id From 25cbb3e553a7b01b6ff184aefc50ae8427c91f72 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 22:39:21 -0400 Subject: [PATCH 024/221] Drop ImplicitCollectionJobs.workflow_invocatin_step relationship --- lib/galaxy/model/__init__.py | 5 +---- test/unit/model/test_mapping.py | 3 --- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 070f6ccc111..fa422e8a91a 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1911,8 +1911,6 @@ class ImplicitCollectionJobs(Base, RepresentById): populated_state = Column(TrimmedString(64), default='new', nullable=False) jobs = relationship('ImplicitCollectionJobsJobAssociation', back_populates='implicit_collection_jobs') - workflow_invocation_step = relationship('WorkflowInvocationStep', - back_populates='implicit_collection_jobs', uselist=False) class populated_states(str, Enum): NEW = 'new' # New implicit jobs object, unpopulated job associations @@ -7059,8 +7057,7 @@ class WorkflowInvocationStep(Base, Dictifiable, RepresentById): workflow_step = relationship('WorkflowStep') job = relationship('Job', back_populates='workflow_invocation_step', uselist=False) - implicit_collection_jobs = relationship('ImplicitCollectionJobs', - back_populates='workflow_invocation_step', uselist=False) + implicit_collection_jobs = relationship('ImplicitCollectionJobs', uselist=False) output_dataset_collections = relationship( 'WorkflowInvocationStepOutputDatasetCollectionAssociation', back_populates='workflow_invocation_step') diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 398252c28b9..cd45bbc2d65 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -2137,16 +2137,13 @@ class TestImplicitCollectionJobs(BaseTest): session, cls_, implicit_collection_jobs_job_association, - workflow_invocation_step, ): obj = cls_() obj.jobs.append(implicit_collection_jobs_job_association) - obj.workflow_invocation_step = workflow_invocation_step with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.jobs == [implicit_collection_jobs_job_association] - assert stored_obj.workflow_invocation_step.id == workflow_invocation_step.id class TestImplicitCollectionJobsJobAssociation(BaseTest): From 08001cad522abba965c6578308fd65951002d3dd Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 15 Sep 2021 22:41:23 -0400 Subject: [PATCH 025/221] Drop UserPreference.user relationship --- lib/galaxy/model/__init__.py | 5 +---- test/unit/model/test_mapping.py | 10 +--------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index fa422e8a91a..7ab450e8589 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -486,9 +486,7 @@ class User(Base, Dictifiable, RepresentById): back_populates='user', cascade='all, delete-orphan', collection_class=ordering_list('order_index')) - _preferences = relationship('UserPreference', - back_populates='user', - collection_class=attribute_mapped_collection('name')) + _preferences = relationship('UserPreference', collection_class=attribute_mapped_collection('name')) values = relationship('FormValues', primaryjoin=(lambda: User.form_values_id == FormValues.id)) # type: ignore # Add type hint (will this work w/SA?) @@ -8515,7 +8513,6 @@ class UserPreference(Base, RepresentById): user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) name = Column(Unicode(255), index=True) value = Column(Text) - user = relationship('User', back_populates='_preferences') def __init__(self, name=None, value=None): # Do not remove this constructor: it is set as the creator for the User.preferences diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index cd45bbc2d65..96407a60c4e 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -5070,7 +5070,7 @@ class TestUserPreference(BaseTest): obj = cls_() obj.name = name obj.value = value - obj.user = user + obj.user_id = user.id with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) @@ -5079,14 +5079,6 @@ class TestUserPreference(BaseTest): assert stored_obj.value == value assert stored_obj.user_id == user.id - def test_relationships(self, session, cls_, user): - obj = cls_() - obj.user = user - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.user.id == user.id - class TestUserQuotaAssociation(BaseTest): From a6f2768afb3d7af12165838aa5bab56b97d95f19 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 17 Sep 2021 15:03:17 -0400 Subject: [PATCH 026/221] Drop JobMetricNumeric.job relationship --- lib/galaxy/model/__init__.py | 3 +-- test/unit/model/test_mapping.py | 10 +--------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 7ab450e8589..770f5d2fdbf 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -823,7 +823,6 @@ class JobMetricNumeric(BaseJobMetric, RepresentById): plugin = Column(Unicode(255)) metric_name = Column(Unicode(255)) metric_value = Column(Numeric(JOB_METRIC_PRECISION, JOB_METRIC_SCALE)) - job = relationship('Job', back_populates='numeric_metrics') class TaskMetricText(BaseJobMetric, RepresentById): @@ -914,7 +913,7 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): output_datasets = relationship('JobToOutputDatasetAssociation', back_populates='job') state_history = relationship('JobStateHistory', back_populates='job') text_metrics = relationship('JobMetricText', back_populates='job') - numeric_metrics = relationship('JobMetricNumeric', back_populates='job') + numeric_metrics = relationship('JobMetricNumeric') job = relationship('GenomeIndexToolData', back_populates='job') interactivetool_entry_points = relationship('InteractiveToolEntryPoint', back_populates='job', uselist=True) diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 96407a60c4e..96cca5d33d2 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -2739,7 +2739,7 @@ class TestJobMetricNumeric(BaseTest): def test_columns(self, session, cls_, job): plugin, metric_name, metric_value = 'a', 'b', 9 obj = cls_(plugin, metric_name, metric_value) - obj.job = job + obj.job_id = job.id with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) @@ -2748,14 +2748,6 @@ class TestJobMetricNumeric(BaseTest): assert stored_obj.plugin == plugin assert stored_obj.metric_value == metric_value - def test_relationships(self, session, cls_, job): - obj = cls_(None, None, None) - obj.job = job - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.job.id == job.id - class TestJobMetricText(BaseTest): From 400bca7a84b9463c6d71fba50a0b449ccad96103 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 17 Sep 2021 17:18:52 -0400 Subject: [PATCH 027/221] Drop LibraryFolder.dataset_collections relationship --- lib/galaxy/model/__init__.py | 3 +-- test/unit/model/test_mapping.py | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 770f5d2fdbf..86f79a242ae 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4494,7 +4494,6 @@ class LibraryFolder(Base, Dictifiable, HasName, RepresentById): lazy=True, viewonly=True) - dataset_collections = relationship('LibraryDatasetCollectionAssociation', back_populates='folder') library_root = relationship('Library', back_populates='root_folder') actions = relationship('LibraryFolderPermissions', back_populates='folder') info_association = relationship('LibraryFolderInfoAssociation', back_populates='folder') @@ -5700,7 +5699,7 @@ class LibraryDatasetCollectionAssociation(Base, DatasetCollectionInstance, Repre deleted = Column(Boolean, default=False) collection = relationship('DatasetCollection') - folder = relationship('LibraryFolder', back_populates='dataset_collections') + folder = relationship('LibraryFolder') tags = relationship('LibraryDatasetCollectionTagAssociation', order_by=lambda: LibraryDatasetCollectionTagAssociation.id, # type: ignore diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 96cca5d33d2..a5e0c050f05 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -3644,7 +3644,6 @@ class TestLibraryFolder(BaseTest): library_folder, library_dataset, library, - library_dataset_collection_association, library_folder_permission, library_folder_info_association, library_folder_factory, @@ -3653,7 +3652,6 @@ class TestLibraryFolder(BaseTest): obj.parent = library_folder folder1 = library_folder_factory() obj.folders.append(folder1) - obj.dataset_collections.append(library_dataset_collection_association) obj.library_root.append(library) obj.actions.append(library_folder_permission) obj.info_association.append(library_folder_info_association) @@ -3672,7 +3670,6 @@ class TestLibraryFolder(BaseTest): # use identity equality instread of object equality. assert stored_obj.datasets[0].id == library_dataset.id assert stored_obj.active_datasets[0].id == library_dataset.id - assert stored_obj.dataset_collections == [library_dataset_collection_association] assert stored_obj.info_association == [library_folder_info_association] delete_from_database(session, folder1) From 8f0d6c595081aed3d264f38477f758ea301fb956 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 17 Sep 2021 17:20:49 -0400 Subject: [PATCH 028/221] Drop LibraryFolder.info_association relationship --- lib/galaxy/model/__init__.py | 4 +--- test/unit/model/test_mapping.py | 9 --------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 86f79a242ae..9c8da783e3a 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4496,7 +4496,6 @@ class LibraryFolder(Base, Dictifiable, HasName, RepresentById): library_root = relationship('Library', back_populates='root_folder') actions = relationship('LibraryFolderPermissions', back_populates='folder') - info_association = relationship('LibraryFolderInfoAssociation', back_populates='folder') dict_element_visible_keys = ['id', 'parent_id', 'name', 'description', 'item_count', 'genome_build', 'update_time', 'deleted'] @@ -4929,8 +4928,7 @@ class LibraryFolderInfoAssociation(Base, RepresentById): folder = relationship('LibraryFolder', primaryjoin=(lambda: (LibraryFolderInfoAssociation.library_folder_id == LibraryFolder.id) # type: ignore - & (not_(LibraryFolderInfoAssociation.deleted))), # type: ignore - back_populates="info_association") + & (not_(LibraryFolderInfoAssociation.deleted)))) # type: ignore template = relationship('FormDefinition', primaryjoin=(lambda: LibraryFolderInfoAssociation.form_definition_id == FormDefinition.id)) # type: ignore info = relationship('FormValues', diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index a5e0c050f05..b9fa918575e 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -3645,7 +3645,6 @@ class TestLibraryFolder(BaseTest): library_dataset, library, library_folder_permission, - library_folder_info_association, library_folder_factory, ): obj = cls_() @@ -3654,7 +3653,6 @@ class TestLibraryFolder(BaseTest): obj.folders.append(folder1) obj.library_root.append(library) obj.actions.append(library_folder_permission) - obj.info_association.append(library_folder_info_association) # There's no back reference, so dataset does not update folder; so we have to flush to the database library_dataset.folder = obj @@ -3670,7 +3668,6 @@ class TestLibraryFolder(BaseTest): # use identity equality instread of object equality. assert stored_obj.datasets[0].id == library_dataset.id assert stored_obj.active_datasets[0].id == library_dataset.id - assert stored_obj.info_association == [library_folder_info_association] delete_from_database(session, folder1) @@ -6978,12 +6975,6 @@ def library_folder(model, session): yield from dbcleanup_wrapper(session, instance) -@pytest.fixture -def library_folder_info_association(model, session, library_folder, form_definition, form_values): - instance = model.LibraryFolderInfoAssociation(library_folder, form_definition, form_values) - yield from dbcleanup_wrapper(session, instance) - - @pytest.fixture def library_folder_permission(model, session, library_folder, role): instance = model.LibraryFolderPermissions('a', library_folder, role) From 86026a0bb8c768ac7b21840ea2799d6e2367f27a Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 17 Sep 2021 17:32:38 -0400 Subject: [PATCH 029/221] Drop Role.library_actions relationship --- lib/galaxy/model/__init__.py | 3 +-- test/unit/model/test_mapping.py | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 9c8da783e3a..1b5f52867ba 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2818,7 +2818,6 @@ class Role(Base, Dictifiable, RepresentById): deleted = Column(Boolean, index=True, default=False) dataset_actions = relationship('DatasetPermissions', back_populates='role') groups = relationship('GroupRoleAssociation', back_populates='role') - library_actions = relationship('LibraryPermissions', back_populates='role') library_folder_actions = relationship('LibraryFolderPermissions', back_populates='role') library_dataset_actions = relationship('LibraryDatasetPermissions', back_populates='role') library_dataset_dataset_actions = relationship( @@ -2980,7 +2979,7 @@ class LibraryPermissions(Base, RepresentById): library_id = Column(Integer, ForeignKey('library.id'), nullable=True, index=True) role_id = Column(Integer, ForeignKey('role.id'), index=True) library = relationship('Library', back_populates='actions') - role = relationship('Role', back_populates='library_actions') + role = relationship('Role') def __init__(self, action, library_item, role): self.action = action diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index b9fa918575e..bda8a39c869 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4304,7 +4304,6 @@ class TestRole(BaseTest): cls_, dataset_permission, group_role_association, - library_permission, library_folder_permission, library_dataset_permission, library_dataset_dataset_association_permission, @@ -4313,7 +4312,6 @@ class TestRole(BaseTest): name, description, type_ = get_unique_value(), 'b', cls_.types.SYSTEM obj = cls_(name, description, type_) obj.dataset_actions.append(dataset_permission) - obj.library_actions.append(library_permission) obj.library_folder_actions.append(library_folder_permission) obj.library_dataset_actions.append(library_dataset_permission) obj.library_dataset_dataset_actions.append(library_dataset_dataset_association_permission) @@ -4324,7 +4322,6 @@ class TestRole(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.dataset_actions == [dataset_permission] assert stored_obj.groups == [group_role_association] - assert stored_obj.library_actions == [library_permission] assert stored_obj.library_folder_actions == [library_folder_permission] assert stored_obj.library_dataset_actions == [library_dataset_permission] assert (stored_obj.library_dataset_dataset_actions From 51e1ea3816d0c9ed868bff788edc954bb9480357 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 17 Sep 2021 17:33:33 -0400 Subject: [PATCH 030/221] Drop Role.library_folder_actions relationship --- lib/galaxy/model/__init__.py | 3 +-- test/unit/model/test_mapping.py | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 1b5f52867ba..c786fc0bcb2 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2818,7 +2818,6 @@ class Role(Base, Dictifiable, RepresentById): deleted = Column(Boolean, index=True, default=False) dataset_actions = relationship('DatasetPermissions', back_populates='role') groups = relationship('GroupRoleAssociation', back_populates='role') - library_folder_actions = relationship('LibraryFolderPermissions', back_populates='role') library_dataset_actions = relationship('LibraryDatasetPermissions', back_populates='role') library_dataset_dataset_actions = relationship( 'LibraryDatasetDatasetAssociationPermissions', back_populates='role') @@ -3000,7 +2999,7 @@ class LibraryFolderPermissions(Base, RepresentById): library_folder_id = Column(Integer, ForeignKey('library_folder.id'), nullable=True, index=True) role_id = Column(Integer, ForeignKey('role.id'), index=True) folder = relationship('LibraryFolder', back_populates='actions') - role = relationship('Role', back_populates='library_folder_actions') + role = relationship('Role') def __init__(self, action, library_item, role): self.action = action diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index bda8a39c869..0a087c4b9f8 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4304,7 +4304,6 @@ class TestRole(BaseTest): cls_, dataset_permission, group_role_association, - library_folder_permission, library_dataset_permission, library_dataset_dataset_association_permission, user_role_association, @@ -4312,7 +4311,6 @@ class TestRole(BaseTest): name, description, type_ = get_unique_value(), 'b', cls_.types.SYSTEM obj = cls_(name, description, type_) obj.dataset_actions.append(dataset_permission) - obj.library_folder_actions.append(library_folder_permission) obj.library_dataset_actions.append(library_dataset_permission) obj.library_dataset_dataset_actions.append(library_dataset_dataset_association_permission) obj.groups.append(group_role_association) @@ -4322,7 +4320,6 @@ class TestRole(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.dataset_actions == [dataset_permission] assert stored_obj.groups == [group_role_association] - assert stored_obj.library_folder_actions == [library_folder_permission] assert stored_obj.library_dataset_actions == [library_dataset_permission] assert (stored_obj.library_dataset_dataset_actions == [library_dataset_dataset_association_permission]) From cfdc25291593487336732b03f8feb3a1b0df9d5e Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 17 Sep 2021 17:34:32 -0400 Subject: [PATCH 031/221] Drop Role.library_dataset_actions relationship --- lib/galaxy/model/__init__.py | 3 +-- test/unit/model/test_mapping.py | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index c786fc0bcb2..af3e8fc093e 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2818,7 +2818,6 @@ class Role(Base, Dictifiable, RepresentById): deleted = Column(Boolean, index=True, default=False) dataset_actions = relationship('DatasetPermissions', back_populates='role') groups = relationship('GroupRoleAssociation', back_populates='role') - library_dataset_actions = relationship('LibraryDatasetPermissions', back_populates='role') library_dataset_dataset_actions = relationship( 'LibraryDatasetDatasetAssociationPermissions', back_populates='role') users = relationship('UserRoleAssociation', back_populates='role') @@ -3020,7 +3019,7 @@ class LibraryDatasetPermissions(Base, RepresentById): library_dataset_id = Column(Integer, ForeignKey('library_dataset.id'), nullable=True, index=True) role_id = Column(Integer, ForeignKey('role.id'), index=True) library_dataset = relationship('LibraryDataset', back_populates='actions') - role = relationship('Role', back_populates='library_dataset_actions') + role = relationship('Role') def __init__(self, action, library_item, role): self.action = action diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 0a087c4b9f8..1009ce5f5ef 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4304,14 +4304,12 @@ class TestRole(BaseTest): cls_, dataset_permission, group_role_association, - library_dataset_permission, library_dataset_dataset_association_permission, user_role_association, ): name, description, type_ = get_unique_value(), 'b', cls_.types.SYSTEM obj = cls_(name, description, type_) obj.dataset_actions.append(dataset_permission) - obj.library_dataset_actions.append(library_dataset_permission) obj.library_dataset_dataset_actions.append(library_dataset_dataset_association_permission) obj.groups.append(group_role_association) obj.users.append(user_role_association) @@ -4320,7 +4318,6 @@ class TestRole(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.dataset_actions == [dataset_permission] assert stored_obj.groups == [group_role_association] - assert stored_obj.library_dataset_actions == [library_dataset_permission] assert (stored_obj.library_dataset_dataset_actions == [library_dataset_dataset_association_permission]) assert stored_obj.users == [user_role_association] From 2cd9eddaa9771a4e93e118c39aeeba89a3dfe0dc Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 17 Sep 2021 17:51:09 -0400 Subject: [PATCH 032/221] Drop Role.library_dataset_dataset_actions relationship --- lib/galaxy/model/__init__.py | 4 +--- test/unit/model/test_mapping.py | 4 ---- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index af3e8fc093e..ce6722f8f57 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2818,8 +2818,6 @@ class Role(Base, Dictifiable, RepresentById): deleted = Column(Boolean, index=True, default=False) dataset_actions = relationship('DatasetPermissions', back_populates='role') groups = relationship('GroupRoleAssociation', back_populates='role') - library_dataset_dataset_actions = relationship( - 'LibraryDatasetDatasetAssociationPermissions', back_populates='role') users = relationship('UserRoleAssociation', back_populates='role') dict_collection_visible_keys = ['id', 'name'] @@ -3042,7 +3040,7 @@ class LibraryDatasetDatasetAssociationPermissions(Base, RepresentById): role_id = Column(Integer, ForeignKey('role.id'), index=True) library_dataset_dataset_association = relationship('LibraryDatasetDatasetAssociation', back_populates='actions') - role = relationship('Role', back_populates='library_dataset_dataset_actions') + role = relationship('Role') def __init__(self, action, library_item, role): self.action = action diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 1009ce5f5ef..360fbfced96 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4304,13 +4304,11 @@ class TestRole(BaseTest): cls_, dataset_permission, group_role_association, - library_dataset_dataset_association_permission, user_role_association, ): name, description, type_ = get_unique_value(), 'b', cls_.types.SYSTEM obj = cls_(name, description, type_) obj.dataset_actions.append(dataset_permission) - obj.library_dataset_dataset_actions.append(library_dataset_dataset_association_permission) obj.groups.append(group_role_association) obj.users.append(user_role_association) @@ -4318,8 +4316,6 @@ class TestRole(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.dataset_actions == [dataset_permission] assert stored_obj.groups == [group_role_association] - assert (stored_obj.library_dataset_dataset_actions - == [library_dataset_dataset_association_permission]) assert stored_obj.users == [user_role_association] From 143bb98c26d58e51f12b6bf5f0361e976f6b1853 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 17 Sep 2021 19:33:30 -0400 Subject: [PATCH 033/221] Fix bug: move non_private_roles to User This relationship was declared on UserRoleAssociation and, via backref, on User. Both relationships were named 'non_private_roles', which was clearly an error. I've moved the relationship to the User, where it belongs; and dropped the reverse on UserRoleAssociation. --- lib/galaxy/model/__init__.py | 18 +++++++----------- test/unit/model/test_mapping.py | 31 +++++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index ce6722f8f57..62e47eb81ca 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -497,6 +497,13 @@ class User(Base, Dictifiable, RepresentById): roles = relationship('UserRoleAssociation', back_populates='user') stored_workflows = relationship('StoredWorkflow', back_populates='user', primaryjoin=(lambda: User.id == StoredWorkflow.user_id)) # type: ignore + non_private_roles = relationship( + 'UserRoleAssociation', + primaryjoin=(lambda: + (User.id == UserRoleAssociation.user_id) # type: ignore + & (UserRoleAssociation.role_id == Role.id) # type: ignore + & not_(Role.name == User.email)) # type: ignore + ) preferences: association_proxy # defined at the end of this module @@ -2774,17 +2781,6 @@ class UserRoleAssociation(Base, RepresentById): user = relationship('User', back_populates="roles") role = relationship('Role', back_populates="users") - # TODO: should be defined on the User model only? - non_private_roles = relationship( - 'User', - backref="non_private_roles", - viewonly=True, - primaryjoin=(lambda: - (User.id == UserRoleAssociation.user_id) # type: ignore - & (UserRoleAssociation.role_id == Role.id) # type: ignore - & not_(Role.name == User.email)) # type: ignore - ) - def __init__(self, user, role): self.user = user self.role = role diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 360fbfced96..bae1d6a3a99 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4834,7 +4834,8 @@ class TestUser(BaseTest): user_preference, api_keys, data_manager_history_association, - user_role_association, + role_factory, + user_role_association_factory, stored_workflow, stored_workflow_menu_entry_factory, ): @@ -4858,6 +4859,14 @@ class TestUser(BaseTest): obj.quotas.append(user_quota_association) obj.social_auth.append(user_authnz_token) + _private_role = role_factory(name=obj.email) + private_user_role = user_role_association_factory(obj, _private_role) + obj.roles.append(private_user_role) + + _non_private_role = role_factory(name='a') + non_private_user_role = user_role_association_factory(obj, _non_private_role) + obj.roles.append(non_private_user_role) + swme = stored_workflow_menu_entry_factory() swme.stored_workflow = stored_workflow swme.user = obj @@ -4867,7 +4876,6 @@ class TestUser(BaseTest): obj.api_keys.append(api_keys) obj.data_manager_histories.append(data_manager_history_association) - obj.roles.append(user_role_association) obj.stored_workflows.append(stored_workflow) with dbcleanup(session, obj) as obj_id: @@ -4887,10 +4895,11 @@ class TestUser(BaseTest): assert user_preference in stored_obj._preferences.values() assert stored_obj.api_keys == [api_keys] assert stored_obj.data_manager_histories == [data_manager_history_association] - assert stored_obj.roles == [user_role_association] + assert are_same_entity_collections(stored_obj.roles, [private_user_role, non_private_user_role]) + assert stored_obj.non_private_roles == [non_private_user_role] assert stored_obj.stored_workflows == [stored_workflow] - delete_from_database(session, [history1, history2, swme]) + delete_from_database(session, [history1, history2, swme, private_user_role, non_private_user_role]) class TestUserAction(BaseTest): @@ -7379,6 +7388,13 @@ def page_rating_association_factory(model): return make_instance +@pytest.fixture +def role_factory(model): + def make_instance(*args, **kwds): + return model.Role(*args, **kwds) + return make_instance + + @pytest.fixture def stored_workflow_menu_entry_factory(model): def make_instance(*args, **kwds): @@ -7400,6 +7416,13 @@ def stored_workflow_tag_association_factory(model): return make_instance +@pytest.fixture +def user_role_association_factory(model): + def make_instance(*args, **kwds): + return model.UserRoleAssociation(*args, **kwds) + return make_instance + + @pytest.fixture def visualization_rating_association_factory(model): def make_instance(*args, **kwds): From 536f255d12b12e506dd1dbf2f883077d8c9b4828 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 17 Sep 2021 19:49:42 -0400 Subject: [PATCH 034/221] Drop JobMetrixText.job relationship --- lib/galaxy/model/__init__.py | 3 +-- test/unit/model/test_mapping.py | 8 -------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 62e47eb81ca..4bdfc6d0aa8 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -819,7 +819,6 @@ class JobMetricText(BaseJobMetric, RepresentById): plugin = Column(Unicode(255)) metric_name = Column(Unicode(255)) metric_value = Column(Unicode(JOB_METRIC_MAX_LENGTH)) - job = relationship('Job', back_populates='text_metrics') class JobMetricNumeric(BaseJobMetric, RepresentById): @@ -919,7 +918,7 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): tasks = relationship('Task', back_populates='job') output_datasets = relationship('JobToOutputDatasetAssociation', back_populates='job') state_history = relationship('JobStateHistory', back_populates='job') - text_metrics = relationship('JobMetricText', back_populates='job') + text_metrics = relationship('JobMetricText') numeric_metrics = relationship('JobMetricNumeric') job = relationship('GenomeIndexToolData', back_populates='job') interactivetool_entry_points = relationship('InteractiveToolEntryPoint', diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index bae1d6a3a99..60927070b56 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -2766,14 +2766,6 @@ class TestJobMetricText(BaseTest): assert stored_obj.plugin == plugin assert stored_obj.metric_value == metric_value - def test_relationships(self, session, cls_, job): - obj = cls_(None, None, None) - obj.job = job - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.job.id == job.id - class TestJobParameter(BaseTest): From 41aac2f04f8db678f81cc6ee21545f6531fb9d01 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 17 Sep 2021 19:51:17 -0400 Subject: [PATCH 035/221] Drop TaskMetricText.task relationship --- lib/galaxy/model/__init__.py | 3 +-- test/unit/model/test_mapping.py | 8 -------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 4bdfc6d0aa8..fb2a42ffcd8 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -839,7 +839,6 @@ class TaskMetricText(BaseJobMetric, RepresentById): plugin = Column(Unicode(255)) metric_name = Column(Unicode(255)) metric_value = Column(Unicode(JOB_METRIC_MAX_LENGTH)) - task = relationship('Task', back_populates='text_metrics') class TaskMetricNumeric(BaseJobMetric, RepresentById): @@ -1572,7 +1571,7 @@ class Task(Base, JobLike, RepresentById): task_runner_external_id = Column(String(255)) prepare_input_files_cmd = Column(TEXT) job = relationship('Job', back_populates='tasks') - text_metrics = relationship('TaskMetricText', back_populates='task') + text_metrics = relationship('TaskMetricText') numeric_metrics = relationship('TaskMetricNumeric', back_populates='task') _numeric_metric = TaskMetricNumeric diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 60927070b56..7949a23489f 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4719,14 +4719,6 @@ class TestTaskMetricText(BaseTest): assert stored_obj.plugin == plugin assert stored_obj.metric_value == metric_value - def test_relationships(self, session, cls_, task): - obj = cls_(None, None, None) - obj.task = task - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.task.id == task.id - class TestToolTagAssociation(BaseTest): From 10db295ba96c28f566dc67b87c28e74efe28ec33 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 17 Sep 2021 19:52:18 -0400 Subject: [PATCH 036/221] Drop TaskMetricNumeric.task relationship --- lib/galaxy/model/__init__.py | 3 +-- test/unit/model/test_mapping.py | 8 -------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index fb2a42ffcd8..40f34dc2b82 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -849,7 +849,6 @@ class TaskMetricNumeric(BaseJobMetric, RepresentById): plugin = Column(Unicode(255)) metric_name = Column(Unicode(255)) metric_value = Column(Numeric(JOB_METRIC_PRECISION, JOB_METRIC_SCALE)) - task = relationship('Task', back_populates='numeric_metrics') class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): @@ -1572,7 +1571,7 @@ class Task(Base, JobLike, RepresentById): prepare_input_files_cmd = Column(TEXT) job = relationship('Job', back_populates='tasks') text_metrics = relationship('TaskMetricText') - numeric_metrics = relationship('TaskMetricNumeric', back_populates='task') + numeric_metrics = relationship('TaskMetricNumeric') _numeric_metric = TaskMetricNumeric _text_metric = TaskMetricText diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 7949a23489f..7eff04a756f 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -4693,14 +4693,6 @@ class TestTaskMetricNumeric(BaseTest): assert stored_obj.plugin == plugin assert stored_obj.metric_value == metric_value - def test_relationships(self, session, cls_, task): - obj = cls_(None, None, None) - obj.task = task - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.task.id == task.id - class TestTaskMetricText(BaseTest): From 9e8668a6dc535c8f9fa341587fe1a461d827fbf9 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 17 Sep 2021 20:03:52 -0400 Subject: [PATCH 037/221] Drop WorkflowInvocation.parent_workflow_invocation_association relationship --- lib/galaxy/model/__init__.py | 7 ------- test/unit/model/test_mapping.py | 8 +------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 40f34dc2b82..f774af2a4f1 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6637,12 +6637,6 @@ class WorkflowInvocation(Base, UsesCreateAndUpdateTime, Dictifiable, RepresentBy back_populates='workflow_invocation') output_datasets = relationship('WorkflowInvocationOutputDatasetAssociation', back_populates='workflow_invocation') - parent_workflow_invocation_association = relationship( - 'WorkflowInvocationToSubworkflowInvocationAssociation', - primaryjoin=(lambda: - WorkflowInvocationToSubworkflowInvocationAssociation.subworkflow_invocation_id # type: ignore - == WorkflowInvocation.id), # type: ignore - back_populates='subworkflow_invocation') output_values = relationship('WorkflowInvocationOutputValue', back_populates='workflow_invocation') dict_collection_visible_keys = ['id', 'update_time', 'create_time', 'workflow_id', 'history_id', 'uuid', 'state'] @@ -7006,7 +7000,6 @@ class WorkflowInvocationToSubworkflowInvocationAssociation(Base, Dictifiable, Re primaryjoin=(lambda: WorkflowInvocationToSubworkflowInvocationAssociation.subworkflow_invocation_id # type: ignore == WorkflowInvocation.id), # type: ignore - back_populates='parent_workflow_invocation_association', uselist=False, ) workflow_step = relationship('WorkflowStep') diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 7eff04a756f..71edc060519 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -5484,8 +5484,6 @@ class TestWorkflowInvocation(BaseTest): ): subworkflow_invocation_assoc = \ workflow_invocation_to_subworkflow_invocation_association_factory() - parent_workflow_invocation_assoc = \ - workflow_invocation_to_subworkflow_invocation_association_factory() obj = cls_() obj.workflow = workflow @@ -5501,7 +5499,6 @@ class TestWorkflowInvocation(BaseTest): obj.output_dataset_collections.append( workflow_invocation_output_dataset_collection_association) obj.output_datasets.append(workflow_invocation_output_dataset_association) - obj.parent_workflow_invocation_association.append(parent_workflow_invocation_assoc) obj.output_values.append(workflow_invocation_output_value) with dbcleanup(session, obj) as obj_id: @@ -5520,12 +5517,9 @@ class TestWorkflowInvocation(BaseTest): assert (stored_obj.output_dataset_collections == [workflow_invocation_output_dataset_collection_association]) assert stored_obj.output_datasets == [workflow_invocation_output_dataset_association] - assert (stored_obj.parent_workflow_invocation_association - == [parent_workflow_invocation_assoc]) assert stored_obj.output_values == [workflow_invocation_output_value] - delete_from_database( - session, [subworkflow_invocation_assoc, parent_workflow_invocation_assoc]) + delete_from_database(session, subworkflow_invocation_assoc) class TestWorkflowInvocationOutputDatasetAssociation(BaseTest): From f740ebfc248fb6efadc66c16676c1e1c7119d367 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 17 Sep 2021 20:06:12 -0400 Subject: [PATCH 038/221] Fix tests for metrics (we don't have job/task relationships, so use id) --- test/unit/model/test_mapping.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 71edc060519..184ef6e3cb8 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -2757,7 +2757,7 @@ class TestJobMetricText(BaseTest): def test_columns(self, session, cls_, job): plugin, metric_name, metric_value = 'a', 'b', 'c' obj = cls_(plugin, metric_name, metric_value) - obj.job = job + obj.job_id = job.id with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) @@ -4684,7 +4684,7 @@ class TestTaskMetricNumeric(BaseTest): def test_columns(self, session, cls_, task): plugin, metric_name, metric_value = 'a', 'b', 9 obj = cls_(plugin, metric_name, metric_value) - obj.task = task + obj.task_id = task.id with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) @@ -4702,7 +4702,7 @@ class TestTaskMetricText(BaseTest): def test_columns(self, session, cls_, task): plugin, metric_name, metric_value = 'a', 'b', 'c' obj = cls_(plugin, metric_name, metric_value) - obj.task = task + obj.task_id = task.id with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) From 2a95fdc0094c6128fddb4b1c1e816c13b7d3c850 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 17 Sep 2021 21:21:52 -0400 Subject: [PATCH 039/221] Fix bug introduced in the fix bug commmit --- lib/galaxy/model/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index f774af2a4f1..e03d24f4ce4 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -499,6 +499,7 @@ class User(Base, Dictifiable, RepresentById): primaryjoin=(lambda: User.id == StoredWorkflow.user_id)) # type: ignore non_private_roles = relationship( 'UserRoleAssociation', + viewonly=True, primaryjoin=(lambda: (User.id == UserRoleAssociation.user_id) # type: ignore & (UserRoleAssociation.role_id == Role.id) # type: ignore From cb061dec19094ef0ee8fcaf9f73c68b5b1a6782c Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 17 Sep 2021 22:49:55 -0400 Subject: [PATCH 040/221] Prevent HistoryAudit from being instantiated The class should never be instantiated. Its instrumented attributes should be only accessed in a class-bound context, where they return SQL expressions, and are used as such in the classmethod `prune`, and in History.update_time column property. Setting __init__ to None guards against accidental instantiation. --- lib/galaxy/model/__init__.py | 6 ++---- test/unit/model/test_mapping.py | 21 --------------------- 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index e03d24f4ce4..cc4393e0c73 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2257,11 +2257,9 @@ class HistoryAudit(Base, RepresentById): history_id = Column(Integer, ForeignKey('history.id'), primary_key=True, nullable=False) update_time = Column(DateTime, default=now, primary_key=True, nullable=False) - history = relationship('History') - def __init__(self, history, update_time): - self.history = history - self.update_time = update_time + __init__ = None # This class should never be instantiated. + # See https://github.com/galaxyproject/galaxy/pull/11914 for details. @classmethod def prune(cls, sa_session): diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 184ef6e3cb8..878acf1a387 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -66,7 +66,6 @@ from uuid import UUID, uuid4 import pytest from sqlalchemy import ( - and_, delete, select, UniqueConstraint, @@ -1462,26 +1461,6 @@ class TestHistoryAnnotationAssociation(BaseTest): assert stored_obj.user.id == user.id -class TestHistoryAudit(BaseTest): - - def test_table(self, cls_): - assert cls_.__tablename__ == 'history_audit' - - def test_columns_and_relationships(self, session, cls_, history): - update_time = datetime.now() - obj = cls_(history, update_time) - - where_clause = and_(cls_.history_id == history.id, cls_.update_time == update_time) - - with dbcleanup(session, obj, where_clause): - stored_obj = get_stored_obj(session, cls_, where_clause=where_clause) - # test columns - assert stored_obj.history_id == history.id - assert stored_obj.update_time == update_time - # test relationships - assert stored_obj.history.id == history.id - - class TestHistoryDatasetAssociation(BaseTest): def test_table(self, cls_): From 52736678dab43f36f4ffccf68c493b5eb0740db2 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 17 Sep 2021 23:21:33 -0400 Subject: [PATCH 041/221] Simplify HistoryAudit.prune Class is declaratively mapped, so we (almost) don't need its table attribute. --- lib/galaxy/model/__init__.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index cc4393e0c73..fc293d4bb9f 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2258,23 +2258,23 @@ class HistoryAudit(Base, RepresentById): history_id = Column(Integer, ForeignKey('history.id'), primary_key=True, nullable=False) update_time = Column(DateTime, default=now, primary_key=True, nullable=False) - __init__ = None # This class should never be instantiated. + # This class should never be instantiated. # See https://github.com/galaxyproject/galaxy/pull/11914 for details. + __init__ = None # type: ignore @classmethod def prune(cls, sa_session): - history_audit_table = cls.table latest_subq = sa_session.query( - history_audit_table.c.history_id, - func.max(history_audit_table.c.update_time).label('max_update_time')).group_by(history_audit_table.c.history_id).subquery() + cls.history_id, + func.max(cls.update_time).label('max_update_time')).group_by(cls.history_id).subquery() not_latest_query = sa_session.query( - history_audit_table.c.history_id, history_audit_table.c.update_time + cls.history_id, cls.update_time ).select_from(latest_subq).join( - history_audit_table, and_( - history_audit_table.c.update_time < latest_subq.columns.max_update_time, - history_audit_table.c.history_id == latest_subq.columns.history_id)) - d = history_audit_table.delete() - sa_session.execute(d.where(tuple_(history_audit_table.c.history_id, history_audit_table.c.update_time).in_(not_latest_query))) + cls, and_( + cls.update_time < latest_subq.columns.max_update_time, + cls.history_id == latest_subq.columns.history_id)) + q = cls.__table__.delete().where(tuple_(cls.history_id, cls.update_time).in_(not_latest_query)) + sa_session.execute(q) class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, RepresentById): From e5110d7fa9c35c3dfba266515721915a8665659d Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 20 Sep 2021 10:38:42 -0400 Subject: [PATCH 042/221] Drop WorkflowStep.parent_workflow_input_connections relationship --- lib/galaxy/model/__init__.py | 4 ---- test/unit/model/test_mapping.py | 2 -- 2 files changed, 6 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index fc293d4bb9f..38b69c9dc34 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6262,9 +6262,6 @@ class WorkflowStep(Base, RepresentById): post_job_actions = relationship('PostJobAction', back_populates='workflow_step') inputs = relationship('WorkflowStepInput', back_populates='workflow_step') workflow_outputs = relationship('WorkflowOutput', back_populates='workflow_step') - parent_workflow_input_connections = relationship('WorkflowStepConnection', - primaryjoin=(lambda: WorkflowStepConnection.input_subworkflow_step_id == WorkflowStep.id) # type: ignore - ) output_connections = relationship('WorkflowStepConnection', primaryjoin=(lambda: WorkflowStepConnection.output_step_id == WorkflowStep.id) # type: ignore ) @@ -6510,7 +6507,6 @@ class WorkflowStepConnection(Base, RepresentById): cascade='all', primaryjoin=(lambda: WorkflowStepConnection.input_step_input_id == WorkflowStepInput.id)) # type: ignore input_subworkflow_step = relationship('WorkflowStep', - back_populates='parent_workflow_input_connections', primaryjoin=(lambda: WorkflowStepConnection.input_subworkflow_step_id == WorkflowStep.id)) # type: ignore output_step = relationship('WorkflowStep', back_populates='output_connections', diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 878acf1a387..22caeca82ff 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -6216,7 +6216,6 @@ class TestWorkflowStep(BaseTest): obj.workflow = workflow obj.subworkflow = subworkflow obj.dynamic_tool = dynamic_tool - obj.parent_workflow_input_connections.append(workflow_step_connection_in) obj.output_connections.append(workflow_step_connection_out) with dbcleanup(session, obj) as obj_id: @@ -6224,7 +6223,6 @@ class TestWorkflowStep(BaseTest): assert stored_obj.workflow.id == workflow.id assert stored_obj.subworkflow.id == subworkflow.id assert stored_obj.dynamic_tool.id == dynamic_tool.id - assert stored_obj.parent_workflow_input_connections == [workflow_step_connection_in] assert stored_obj.output_connections == [workflow_step_connection_out] persisted = [subworkflow, workflow_step_connection_in, workflow_step_connection_out] From ed68b8726633499c93e3f4fcf3ee17744319bb95 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 20 Sep 2021 11:00:13 -0400 Subject: [PATCH 043/221] Drop HDCA.output_dataset_collection_instances relationship --- lib/galaxy/model/__init__.py | 9 +-------- test/unit/model/test_mapping.py | 3 --- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 38b69c9dc34..ba5f9972c29 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1808,10 +1808,7 @@ class JobToOutputDatasetCollectionAssociation(Base, RepresentById): dataset_collection_id = Column(Integer, ForeignKey('history_dataset_collection_association.id'), index=True) name = Column(Unicode(255)) - dataset_collection_instance = relationship( - 'HistoryDatasetCollectionAssociation', - lazy=False, - back_populates="output_dataset_collection_instances") + dataset_collection_instance = relationship('HistoryDatasetCollectionAssociation', lazy=False) job = relationship('Job', back_populates='output_dataset_collection_instances') def __init__(self, name, dataset_collection_instance): @@ -5476,10 +5473,6 @@ class HistoryDatasetCollectionAssociation( order_by=lambda: HistoryDatasetCollectionRatingAssociation.id, # type: ignore back_populates='dataset_collection', ) - output_dataset_collection_instances = relationship( - 'JobToOutputDatasetCollectionAssociation', - back_populates='dataset_collection_instance', - ) hidden_dataset_instances = relationship('HistoryDatasetAssociation', primaryjoin=(lambda: HistoryDatasetAssociation.table.c.hidden_beneath_collection_instance_id # type: ignore == HistoryDatasetCollectionAssociation.id), # type: ignore diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 22caeca82ff..64e103dd9b8 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -1902,7 +1902,6 @@ class TestHistoryDatasetCollectionAssociation(BaseTest): obj.tags.append(history_dataset_collection_tag_association) obj.annotations.append(history_dataset_collection_annotation_association) obj.ratings.append(history_dataset_collection_rating_association) - obj.output_dataset_collection_instances.append(job_to_output_dataset_collection_association) obj.hidden_dataset_instances.append(history_dataset_association) with dbcleanup(session, obj) as obj_id: @@ -1918,8 +1917,6 @@ class TestHistoryDatasetCollectionAssociation(BaseTest): assert stored_obj.tags == [history_dataset_collection_tag_association] assert stored_obj.annotations == [history_dataset_collection_annotation_association] assert stored_obj.ratings == [history_dataset_collection_rating_association] - assert (stored_obj.output_dataset_collection_instances - == [job_to_output_dataset_collection_association]) assert stored_obj.job_state_summary # this is a view; TODO: can we test this better? assert stored_obj.hidden_dataset_instances == [history_dataset_association] From d41ecff1fe06d9af92fd12528e3b910cf6e595dc Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 20 Sep 2021 11:03:16 -0400 Subject: [PATCH 044/221] Drop HDCS.hidden_dataset_instances relationship --- lib/galaxy/model/__init__.py | 7 +------ test/unit/model/test_mapping.py | 3 --- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index ba5f9972c29..8133adeb788 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -5473,10 +5473,6 @@ class HistoryDatasetCollectionAssociation( order_by=lambda: HistoryDatasetCollectionRatingAssociation.id, # type: ignore back_populates='dataset_collection', ) - hidden_dataset_instances = relationship('HistoryDatasetAssociation', - primaryjoin=(lambda: HistoryDatasetAssociation.table.c.hidden_beneath_collection_instance_id # type: ignore - == HistoryDatasetCollectionAssociation.id), # type: ignore - back_populates='hidden_beneath_collection_instance') editable_keys = ('name', 'deleted', 'visible') @@ -8787,8 +8783,7 @@ mapper_registry.map_imperatively( hidden_beneath_collection_instance=relationship(HistoryDatasetCollectionAssociation, primaryjoin=(HistoryDatasetAssociation.table.c.hidden_beneath_collection_instance_id == HistoryDatasetCollectionAssociation.id), - uselist=False, - back_populates="hidden_dataset_instances"), + uselist=False), _metadata=deferred(HistoryDatasetAssociation.table.c._metadata), dependent_jobs=relationship(JobToInputDatasetAssociation, back_populates='dataset'), creating_job_associations=relationship( diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 64e103dd9b8..492a356a9d9 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -1887,7 +1887,6 @@ class TestHistoryDatasetCollectionAssociation(BaseTest): history_dataset_collection_rating_association, history_dataset_collection_tag_association, job_to_output_dataset_collection_association, - history_dataset_association, ): copied_to_hdca = history_dataset_collection_association_factory() @@ -1902,7 +1901,6 @@ class TestHistoryDatasetCollectionAssociation(BaseTest): obj.tags.append(history_dataset_collection_tag_association) obj.annotations.append(history_dataset_collection_annotation_association) obj.ratings.append(history_dataset_collection_rating_association) - obj.hidden_dataset_instances.append(history_dataset_association) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) @@ -1918,7 +1916,6 @@ class TestHistoryDatasetCollectionAssociation(BaseTest): assert stored_obj.annotations == [history_dataset_collection_annotation_association] assert stored_obj.ratings == [history_dataset_collection_rating_association] assert stored_obj.job_state_summary # this is a view; TODO: can we test this better? - assert stored_obj.hidden_dataset_instances == [history_dataset_association] delete_from_database(session, copied_to_hdca) From 20b47336c2559d8bd4c0096a5874293ffe03a4cf Mon Sep 17 00:00:00 2001 From: Alexander OSTROVSKY Date: Mon, 20 Sep 2021 11:28:32 -0700 Subject: [PATCH 045/221] fix cmap sniffer --- lib/galaxy/datatypes/tabular.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 328a76c48f0..5ec6be373ef 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -1362,6 +1362,12 @@ class CMAP(TabularData): file_ext = "cmap" def sniff_prefix(self, file_prefix): + start = file_prefix.string_io().read(3000).strip().split('\n') + for line in start: + if '# CMAP File Version' in line: + return True + else: + pass return file_prefix.startswith('# CMAP File Version:') def set_meta(self, dataset, overwrite=True, skip=None, max_data_lines=7, **kwd): From eca9a4c90106c52e979c467dd42a68ec035bdd78 Mon Sep 17 00:00:00 2001 From: Alex Ostrovsky <40246333+astrovsky01@users.noreply.github.com> Date: Mon, 20 Sep 2021 12:05:01 -0700 Subject: [PATCH 046/221] Update lib/galaxy/datatypes/tabular.py Co-authored-by: Nicola Soranzo --- lib/galaxy/datatypes/tabular.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 5ec6be373ef..6b0d1b113d3 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -1362,13 +1362,13 @@ class CMAP(TabularData): file_ext = "cmap" def sniff_prefix(self, file_prefix): - start = file_prefix.string_io().read(3000).strip().split('\n') - for line in start: - if '# CMAP File Version' in line: + handle = file_prefix.string_io() + for line in handle: + if not line.startswith('#'): + return False + if line.startswith('# CMAP File Version:') return True - else: - pass - return file_prefix.startswith('# CMAP File Version:') + return False def set_meta(self, dataset, overwrite=True, skip=None, max_data_lines=7, **kwd): if dataset.has_data(): From 088f50bea059fe8ed7f3ec6b0fa20dc835cfc70d Mon Sep 17 00:00:00 2001 From: Alexander OSTROVSKY Date: Mon, 20 Sep 2021 12:08:38 -0700 Subject: [PATCH 047/221] missing colon --- lib/galaxy/datatypes/tabular.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 6b0d1b113d3..e9a74f29995 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -1366,7 +1366,7 @@ class CMAP(TabularData): for line in handle: if not line.startswith('#'): return False - if line.startswith('# CMAP File Version:') + if line.startswith('# CMAP File Version:'): return True return False From fe56317263cab5b8ebe183c382337379e5b6fceb Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Mon, 26 Jul 2021 21:22:54 -0400 Subject: [PATCH 048/221] Bypass problematic threads.js / worker --- .../src/components/History/caching/index.js | 72 ++++++++++++------- 1 file changed, 45 insertions(+), 27 deletions(-) diff --git a/client/src/components/History/caching/index.js b/client/src/components/History/caching/index.js index 879444a50b2..1dd1c91f6aa 100644 --- a/client/src/components/History/caching/index.js +++ b/client/src/components/History/caching/index.js @@ -1,32 +1,50 @@ // Exposes worker functions as promises or Observable operators -import { toPromise, toOperator } from "./workerClient"; -/** - * monitor cache for changes - */ -export const monitorContentQuery = toOperator("monitorContentQuery"); -export const monitorDscQuery = toOperator("monitorDscQuery"); -export const monitorHistoryContent = toOperator("monitorHistoryContent"); -export const monitorCollectionContent = toOperator("monitorCollectionContent"); +export { monitorContentQuery, monitorDscQuery, monitorHistoryContent, monitorCollectionContent } from "./CacheApi"; -/** - * Loaders - */ -export const loadHistoryContents = toOperator("loadHistoryContents"); -export const loadDscContent = toOperator("loadDscContent"); +export { loadHistoryContents, loadDscContent } from "./CacheApi"; -/** - * Cache promise functions - */ -export const cacheContent = toPromise("cacheContent"); -export const getCachedContent = toPromise("getCachedContent"); -export const uncacheContent = toPromise("uncacheContent"); -export const bulkCacheContent = toPromise("bulkCacheContent"); -export const cacheCollectionContent = toPromise("cacheCollectionContent"); -export const getCachedCollectionContent = toPromise("getCachedCollectionContent"); -export const bulkCacheDscContent = toPromise("bulkCacheDscContent"); -export const getContentByTypeId = toPromise("getContentByTypeId"); +export { + cacheContent, + getCachedContent, + uncacheContent, + bulkCacheContent, + cacheCollectionContent, + getCachedCollectionContent, + bulkCacheDscContent, + getContentByTypeId, +} from "./CacheApi"; -// Debugging -export const wipeDatabase = toPromise("wipeDatabase"); -export const clearHistoryDateStore = toPromise("clearHistoryDateStore"); +export { wipeDatabase, clearHistoryDateStore } from "./CacheApi"; + +// TODO: The above exports bypass the worker completely for now, swap back to below to use. +//import { toPromise, toOperator } from "./workerClient"; +///** +// * monitor cache for changes +// */ +//export const monitorContentQuery = toOperator("monitorContentQuery"); +//export const monitorDscQuery = toOperator("monitorDscQuery"); +//export const monitorHistoryContent = toOperator("monitorHistoryContent"); +//export const monitorCollectionContent = toOperator("monitorCollectionContent"); +// +///** +// * Loaders +// */ +//export const loadHistoryContents = toOperator("loadHistoryContents"); +//export const loadDscContent = toOperator("loadDscContent"); +// +///** +// * Cache promise functions +// */ +//export const cacheContent = toPromise("cacheContent"); +//export const getCachedContent = toPromise("getCachedContent"); +//export const uncacheContent = toPromise("uncacheContent"); +//export const bulkCacheContent = toPromise("bulkCacheContent"); +//export const cacheCollectionContent = toPromise("cacheCollectionContent"); +//export const getCachedCollectionContent = toPromise("getCachedCollectionContent"); +//export const bulkCacheDscContent = toPromise("bulkCacheDscContent"); +//export const getContentByTypeId = toPromise("getContentByTypeId"); +// +//// Debugging +//export const wipeDatabase = toPromise("wipeDatabase"); +//export const clearHistoryDateStore = toPromise("clearHistoryDateStore"); From 1e59b4a727947af4b5889891681da3c825c78f86 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Mon, 20 Sep 2021 21:48:34 -0400 Subject: [PATCH 049/221] Drop worker-loader --- client/package.json | 1 - client/webpack.config.js | 4 ---- client/yarn.lock | 29 +---------------------------- 3 files changed, 1 insertion(+), 33 deletions(-) diff --git a/client/package.json b/client/package.json index 053fc60a063..2d5c06a28a2 100644 --- a/client/package.json +++ b/client/package.json @@ -169,7 +169,6 @@ "webpack-cli": "^3.3.11", "webpack-dev-server": "^3.11.0", "webpack-merge": "^4.2.2", - "worker-loader": "^3.0.6", "yaml-jest": "^1.0.5", "yaml-loader": "^0.6.0" } diff --git a/client/webpack.config.js b/client/webpack.config.js index 4eb2694f59c..0c2633b0970 100644 --- a/client/webpack.config.js +++ b/client/webpack.config.js @@ -183,10 +183,6 @@ module.exports = (env = {}, argv = {}) => { test: /\.(txt|tmpl)$/, loader: "raw-loader", }, - { - test: /\.worker\.js$/, - use: { loader: "worker-loader" }, - }, ], }, node: { diff --git a/client/yarn.lock b/client/yarn.lock index a42d657c21f..843f5f52780 100644 --- a/client/yarn.lock +++ b/client/yarn.lock @@ -1963,11 +1963,6 @@ jest-diff "^25.2.1" pretty-format "^25.2.1" -"@types/json-schema@^7.0.6": - version "7.0.6" - resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.6.tgz#f4c7ec43e81b319a9815115031709f26987891f0" - integrity sha512-3c+yGKvVP5Y9TYBEibGNR+kLtijnj7mYrXRg+WpFb2X9xm04g/DXYkfg4hmzJQosc9snFNUPkbYIhu+KAm6jJw== - "@types/minimatch@*": version "3.0.3" resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d" @@ -2443,11 +2438,6 @@ ajv-keywords@^3.4.1: resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.4.1.tgz#ef916e271c64ac12171fd8384eaae6b2345854da" integrity sha512-RO1ibKvd27e6FEShVFfPALuHI3WjSVNeK5FIsmme/LYRNxjKuNj+Dt7bucLa6NdSv3JcVTyMlm9kGR84z1XpaQ== -ajv-keywords@^3.5.2: - version "3.5.2" - resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.5.2.tgz#31f29da5ab6e00d1c2d329acf7b5929614d5014d" - integrity sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ== - ajv@^6.1.0: version "6.5.4" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.5.4.tgz#247d5274110db653706b550fcc2b797ca28cfc59" @@ -2478,7 +2468,7 @@ ajv@^6.12.0, ajv@^6.5.5: json-schema-traverse "^0.4.1" uri-js "^4.2.2" -ajv@^6.12.4, ajv@^6.12.5: +ajv@^6.12.4: version "6.12.6" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== @@ -13152,15 +13142,6 @@ schema-utils@^2.6.5, schema-utils@^2.6.6: ajv "^6.12.0" ajv-keywords "^3.4.1" -schema-utils@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-3.0.0.tgz#67502f6aa2b66a2d4032b4279a2944978a0913ef" - integrity sha512-6D82/xSzO094ajanoOSbe4YvXWMfn2A//8Y1+MUqFAJul5Bs+yn36xbK9OtNDcRVSBJ9jjeoXftM6CfztsjOAA== - dependencies: - "@types/json-schema" "^7.0.6" - ajv "^6.12.5" - ajv-keywords "^3.5.2" - select-hose@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/select-hose/-/select-hose-2.0.0.tgz#625d8658f865af43ec962bfc376a37359a4994ca" @@ -15677,14 +15658,6 @@ worker-farm@^1.7.0: dependencies: errno "~0.1.7" -worker-loader@^3.0.6: - version "3.0.6" - resolved "https://registry.yarnpkg.com/worker-loader/-/worker-loader-3.0.6.tgz#fa540eaa806422b744ddcd64db7eced7ae32a6ff" - integrity sha512-yLmxR1momXc8R8NM4j4/nq7hPvVTems7i40NuAdVmitLJwq4agIeZRTyW2Hans0J21laF0e6/ypYbwLpBzlifA== - dependencies: - loader-utils "^2.0.0" - schema-utils "^3.0.0" - worker-rpc@^0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/worker-rpc/-/worker-rpc-0.1.1.tgz#cb565bd6d7071a8f16660686051e969ad32f54d5" From 0eebd3ec3dec9c539d56112fa503a25f02cd63df Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 25 Aug 2021 16:29:25 +0200 Subject: [PATCH 050/221] Refactor SerializationParams dict into a model --- lib/galaxy/managers/configuration.py | 4 +- lib/galaxy/managers/histories.py | 10 ++-- lib/galaxy/managers/history_contents.py | 4 +- lib/galaxy/schema/__init__.py | 26 +++++++++ lib/galaxy/schema/types.py | 9 --- lib/galaxy/webapps/galaxy/api/common.py | 18 +++++- .../webapps/galaxy/api/configuration.py | 2 +- .../webapps/galaxy/api/history_contents.py | 55 +++++++++---------- 8 files changed, 80 insertions(+), 48 deletions(-) diff --git a/lib/galaxy/managers/configuration.py b/lib/galaxy/managers/configuration.py index 7c893ff20e4..69e14308ee6 100644 --- a/lib/galaxy/managers/configuration.py +++ b/lib/galaxy/managers/configuration.py @@ -18,8 +18,8 @@ from typing import ( from galaxy.app import MinimalManagerApp from galaxy.managers import base from galaxy.managers.context import ProvidesUserContext +from galaxy.schema import SerializationParams from galaxy.schema.fields import EncodedDatabaseIdField -from galaxy.schema.types import SerializationParams from galaxy.web.framework.base import server_starttime log = logging.getLogger(__name__) @@ -42,7 +42,7 @@ class ConfigurationManager: host = getattr(trans, "host", None) serializer_class = AdminConfigSerializer if is_admin else ConfigSerializer serializer = serializer_class(self._app) - return serializer.serialize_to_view(self._app.config, host=host, **serialization_params) + return serializer.serialize_to_view(self._app.config, host=host, **serialization_params.dict()) def version(self) -> Dict[str, Any]: version_info = { diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index d69fcfbb454..ff6319120d5 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -44,7 +44,10 @@ from galaxy.managers.base import ( ) from galaxy.managers.citations import CitationsManager from galaxy.managers.users import UserManager -from galaxy.schema import FilterQueryParams +from galaxy.schema import ( + FilterQueryParams, + SerializationParams, +) from galaxy.schema.fields import EncodedDatabaseIdField from galaxy.schema.schema import ( CreateHistoryPayload, @@ -59,7 +62,6 @@ from galaxy.schema.schema import ( JobImportHistoryResponse, LabelValuePair, ) -from galaxy.schema.types import SerializationParams from galaxy.security.idencoding import IdEncodingHelper from galaxy.structured_app import MinimalManagerApp from galaxy.util import restore_text @@ -1066,11 +1068,11 @@ class HistoriesService(ServiceBase): Returns a dictionary with the corresponding values depending on the serialization parameters provided. """ - serialization_params["default_view"] = default_view + serialization_params.default_view = default_view serialized_history = self.serializer.serialize_to_view( history, user=trans.user, trans=trans, - **serialization_params + **serialization_params.dict() ) return serialized_history diff --git a/lib/galaxy/managers/history_contents.py b/lib/galaxy/managers/history_contents.py index e149b85f429..1281655e7d1 100644 --- a/lib/galaxy/managers/history_contents.py +++ b/lib/galaxy/managers/history_contents.py @@ -411,8 +411,8 @@ class HistoryContentsManager(containers.ContainerManagerMixin, base.SortableMana # This will conditionally join a potentially costly job_state summary # All the paranoia if-checking makes me wonder if serialization_params # should really be a property of the manager class instance - if serialization_params and serialization_params['keys']: - if 'job_state_summary' in serialization_params['keys']: + if serialization_params and serialization_params.keys: + if 'job_state_summary' in serialization_params.keys: query = query.options(eagerload('job_state_summary')) return {row.id: row for row in query.all()} diff --git a/lib/galaxy/schema/__init__.py b/lib/galaxy/schema/__init__.py index 1159a5a9ddd..c8a40e0afaf 100644 --- a/lib/galaxy/schema/__init__.py +++ b/lib/galaxy/schema/__init__.py @@ -56,3 +56,29 @@ class FilterQueryParams(BaseModel): ), example="name-dsc,create_time", ) + + +class SerializationParams(BaseModel): + """Contains common parameters for customizing model serialization.""" + view: Optional[str] = Field( + default=None, + title='View', + description=( + 'The name of the view used to serialize this item. ' + 'This will return a predefined set of attributes of the item.' + ), + example="summary" + ) + keys: Optional[List[str]] = Field( + default=None, + title='Keys', + description=( + 'List of keys (name of the attributes) that will be returned in addition ' + 'to the ones included in the `view`.' + ), + ) + default_view: Optional[str] = Field( + default=None, + title='Default View', + description='The item view that will be used in case none was specified.', + ) diff --git a/lib/galaxy/schema/types.py b/lib/galaxy/schema/types.py index 025a780397c..803ab85f459 100644 --- a/lib/galaxy/schema/types.py +++ b/lib/galaxy/schema/types.py @@ -1,12 +1,3 @@ -from typing import ( - Dict, - List, - Optional, - Union, -) - -SerializationParams = Dict[str, Optional[Union[str, List]]] - # Relative URLs cannot be validated with AnyUrl, they need a scheme. # Making them an alias of `str` for now RelativeUrl = str diff --git a/lib/galaxy/webapps/galaxy/api/common.py b/lib/galaxy/webapps/galaxy/api/common.py index f8bc267c2c6..0d6638e6cd5 100644 --- a/lib/galaxy/webapps/galaxy/api/common.py +++ b/lib/galaxy/webapps/galaxy/api/common.py @@ -3,7 +3,7 @@ from typing import Optional from fastapi import Query -from galaxy.schema.types import SerializationParams +from galaxy.schema import SerializationParams SerializationViewQueryParam: Optional[str] = Query( None, @@ -17,6 +17,12 @@ SerializationKeysQueryParam: Optional[str] = Query( description='Comma-separated list of keys to be passed to the serializer', ) +SerializationDefaultViewQueryParam: Optional[str] = Query( + None, + title='Default View', + description='The item view that will be used in case no particular view was specified.', +) + def parse_serialization_params( view: Optional[str] = None, @@ -27,4 +33,12 @@ def parse_serialization_params( key_list = None if keys: key_list = keys.split(',') - return dict(view=view, keys=key_list, default_view=default_view) + return SerializationParams(view=view, keys=key_list, default_view=default_view) + + +def query_serialization_params( + view: Optional[str] = SerializationViewQueryParam, + keys: Optional[str] = SerializationKeysQueryParam, + default_view: Optional[str] = SerializationDefaultViewQueryParam, +) -> SerializationParams: + return parse_serialization_params(view=view, keys=keys, default_view=default_view) diff --git a/lib/galaxy/webapps/galaxy/api/configuration.py b/lib/galaxy/webapps/galaxy/api/configuration.py index 5c7de4f4ad5..4c1d9c9eb73 100644 --- a/lib/galaxy/webapps/galaxy/api/configuration.py +++ b/lib/galaxy/webapps/galaxy/api/configuration.py @@ -211,4 +211,4 @@ def _user_to_model(user, security): def _index(manager, trans, view, keys): serialization_params = parse_serialization_params(view, keys, 'all') - return manager.get_configuration(trans, serialization_params) + return manager.get_configuration(trans, serialization_params.dict()) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index b7c4370d376..4f235aa4ba9 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -48,7 +48,10 @@ from galaxy.model import ( LibraryDataset, ) from galaxy.model.security import GalaxyRBACAgent -from galaxy.schema import FilterQueryParams +from galaxy.schema import ( + FilterQueryParams, + SerializationParams, +) from galaxy.schema.fields import ( EncodedDatabaseIdField, Field, @@ -75,7 +78,6 @@ from galaxy.schema.schema import ( UpdateHistoryContentsBatchPayload, WorkflowInvocationStateSummary, ) -from galaxy.schema.types import SerializationParams from galaxy.security.idencoding import IdEncodingHelper from galaxy.util.json import safe_dumps from galaxy.util.zipstream import ZipstreamWrapper @@ -578,9 +580,9 @@ class HistoriesContentsService(ServiceBase): rval = [] for hda in hdas: self.__deserialize_dataset(trans, hda, payload_dict) - serialization_params["default_view"] = "summary" + serialization_params.default_view = "summary" rval.append(self.hda_serializer.serialize_to_view( - hda, user=trans.user, trans=trans, **serialization_params + hda, user=trans.user, trans=trans, **serialization_params.dict() )) for hdca_id in hdca_ids: self.__update_dataset_collection(trans, hdca_id, payload.dict(exclude_defaults=True)) @@ -795,9 +797,6 @@ class HistoriesContentsService(ServiceBase): trans.response.status = 204 return - # parse content params - view = serialization_params.pop('view') - # SEEK UP, contents > hid up_params = filter_params + self._hid_greater_than(hid) up_order = 'hid-asc' @@ -812,9 +811,9 @@ class HistoriesContentsService(ServiceBase): min_hid, max_hid = self._get_filtered_extrema(history, filter_params) # results - up = self._expand_contents(trans, contents_up, serialization_params, view) + up = self._expand_contents(trans, contents_up, serialization_params) up.reverse() - down = self._expand_contents(trans, contents_down, serialization_params, view) + down = self._expand_contents(trans, contents_down, serialization_params) contents = up + down # Put stats in http headers @@ -857,16 +856,16 @@ class HistoriesContentsService(ServiceBase): # Adds subquery details to initial contents results, perhaps better realized # as a proc or view. - def _expand_contents(self, trans, contents, serialization_params, view): + def _expand_contents(self, trans, contents, serialization_params: SerializationParams): rval = [] for content in contents: if isinstance(content, HistoryDatasetAssociation): dataset = self.hda_serializer.serialize_to_view(content, - user=trans.user, trans=trans, view=view, **serialization_params) + user=trans.user, trans=trans, **serialization_params.dict()) rval.append(dataset) elif isinstance(content, HistoryDatasetCollectionAssociation): collection = self.hdca_serializer.serialize_to_view(content, - user=trans.user, trans=trans, view=view, **serialization_params) + user=trans.user, trans=trans, **serialization_params.dict()) rval.append(collection) return rval @@ -910,9 +909,9 @@ class HistoriesContentsService(ServiceBase): self.hda_manager.purge(hda) else: self.hda_manager.delete(hda) - serialization_params["default_view"] = 'detailed' + serialization_params.default_view = 'detailed' return self.hda_serializer.serialize_to_view( - hda, user=trans.user, trans=trans, **serialization_params + hda, user=trans.user, trans=trans, **serialization_params.dict() ) def __update_dataset_collection(self, trans, id: EncodedDatabaseIdField, payload: Dict[str, Any]): @@ -934,9 +933,9 @@ class HistoriesContentsService(ServiceBase): hda = self.__datasets_for_update(trans, history, [decoded_id], payload)[0] if hda: self.__deserialize_dataset(trans, hda, payload) - serialization_params["default_view"] = 'detailed' + serialization_params.default_view = 'detailed' return self.hda_serializer.serialize_to_view( - hda, user=trans.user, trans=trans, **serialization_params + hda, user=trans.user, trans=trans, **serialization_params.dict() ) return {} @@ -1004,7 +1003,6 @@ class HistoriesContentsService(ServiceBase): # TODO: > 16.04: remove these # TODO: remove 'dataset_details' and the following section when the UI doesn't need it parsed_legacy_params = self._parse_legacy_contents_params(legacy_params) - contents = self.history_contents_manager.contents( history, filters=filters, @@ -1049,7 +1047,8 @@ class HistoriesContentsService(ServiceBase): Returns a dictionary with the appropriate values depending on the serialization parameters provided. """ - view = serialization_params.pop("view", default_view) or default_view + serialization_params_dict = serialization_params.dict() + view = serialization_params_dict.pop("view", default_view) or default_view serializer: Optional[ModelSerializer] = None if isinstance(content, HistoryDatasetAssociation): @@ -1063,7 +1062,7 @@ class HistoriesContentsService(ServiceBase): raise exceptions.UnknownContentsType(f'Unknown contents type: {content.content_type}') return serializer.serialize_to_view( - content, user=trans.user, trans=trans, view=view, **serialization_params + content, user=trans.user, trans=trans, view=view, **serialization_params_dict ) def _parse_legacy_contents_params(self, params: HistoryContentsIndexLegacyParams): @@ -1104,10 +1103,10 @@ class HistoriesContentsService(ServiceBase): id: EncodedDatabaseIdField, serialization_params: SerializationParams, ): - serialization_params["default_view"] = "detailed" + serialization_params.default_view = "detailed" hda = self.hda_manager.get_accessible(self.decode_id(id), trans.user) return self.hda_serializer.serialize_to_view( - hda, user=trans.user, trans=trans, **serialization_params + hda, user=trans.user, trans=trans, **serialization_params.dict() ) def __show_dataset_collection( @@ -1117,7 +1116,7 @@ class HistoriesContentsService(ServiceBase): fuzzy_count: Optional[int] = None, ): dataset_collection_instance = self.__get_accessible_collection(trans, id) - view = serialization_params.get("view") or "element" + view = serialization_params.view or "element" return self.__collection_dict(trans, dataset_collection_instance, view=view, fuzzy_count=fuzzy_count) def __get_accessible_collection(self, trans, id: EncodedDatabaseIdField): @@ -1165,7 +1164,7 @@ class HistoriesContentsService(ServiceBase): for ld in traverse(folder): hda = ld.library_dataset_dataset_association.to_history_dataset_association(history, add_to_history=True) hda_dict = self.hda_serializer.serialize_to_view( - hda, user=trans.user, trans=trans, default_view='detailed', **serialization_params + hda, user=trans.user, trans=trans, default_view='detailed', **serialization_params.dict() ) rval.append(hda_dict) else: @@ -1199,9 +1198,9 @@ class HistoriesContentsService(ServiceBase): return None trans.sa_session.flush() - serialization_params["default_view"] = 'detailed' + serialization_params.default_view = 'detailed' return self.hda_serializer.serialize_to_view( - hda, user=trans.user, trans=trans, **serialization_params + hda, user=trans.user, trans=trans, **serialization_params.dict() ) def __create_hda_from_ldda(self, trans, history: History, ldda_id: EncodedDatabaseIdField): @@ -1298,10 +1297,10 @@ class HistoriesContentsService(ServiceBase): raise exceptions.RequestParameterInvalidException(message) # if the consumer specified keys or view, use the secondary serializer - if serialization_params.get('view') or serialization_params.get('keys'): - serialization_params["default_view"] = 'detailed' + if serialization_params.view or serialization_params.keys: + serialization_params.default_view = 'detailed' return self.hdca_serializer.serialize_to_view( - dataset_collection_instance, user=trans.user, trans=trans, **serialization_params + dataset_collection_instance, user=trans.user, trans=trans, **serialization_params.dict() ) return self.__collection_dict(trans, dataset_collection_instance, view="element") From 6298f8d5349811457c5e713601f65539b849872b Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 26 Aug 2021 21:11:20 +0200 Subject: [PATCH 051/221] Add explicit exception when trying to create a history as anonymous This way it will also work in FastAPI. Also fixed a couple of type hints. --- lib/galaxy/managers/histories.py | 20 ++++++++++++++------ lib/galaxy/schema/schema.py | 6 +++++- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index ff6319120d5..b06b42b25ca 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -43,6 +43,7 @@ from galaxy.managers.base import ( SortableManager, ) from galaxy.managers.citations import CitationsManager +from galaxy.managers.context import ProvidesHistoryContext from galaxy.managers.users import UserManager from galaxy.schema import ( FilterQueryParams, @@ -57,7 +58,7 @@ from galaxy.schema.schema import ( HistoryDetailed, HistoryImportArchiveSourceType, HistorySummary, - JobExportHistoryArchive, + JobExportHistoryArchiveModel, JobIdResponse, JobImportHistoryResponse, LabelValuePair, @@ -690,7 +691,7 @@ class HistoriesService(ServiceBase): serialization_params: SerializationParams, filter_query_params: FilterQueryParams, deleted_only: Optional[bool] = False, - all_histories: bool = False, + all_histories: Optional[bool] = False, ): """ Return a collection of histories for the current user. Additional filters can be applied. @@ -759,13 +760,15 @@ class HistoriesService(ServiceBase): def create( self, - trans, + trans: ProvidesHistoryContext, payload: CreateHistoryPayload, serialization_params: SerializationParams, ): """Create a new history from scratch, by copying an existing one or by importing from URL or File depending on the provided parameters in the payload. """ + if trans.anonymous: + raise glx_exceptions.AuthenticationRequired("You need to be logged in to create histories.") if trans.user and trans.user.bootstrap_admin_user: raise glx_exceptions.RealUserRequiredException("Only real users can create histories.") hist_name = None @@ -982,7 +985,12 @@ class HistoriesService(ServiceBase): """ return self.history_export_view.get_exports(trans, id) - def archive_export(self, trans, id: EncodedDatabaseIdField, payload: ExportHistoryArchivePayload) -> Union[JobExportHistoryArchive, JobIdResponse]: + def archive_export( + self, + trans, + id: EncodedDatabaseIdField, + payload: ExportHistoryArchivePayload, + ) -> Union[JobExportHistoryArchiveModel, JobIdResponse]: """ start job (if needed) to create history export for corresponding history. @@ -1022,13 +1030,13 @@ class HistoriesService(ServiceBase): if up_to_date and jeha.ready: serialized_jeha = self.history_export_view.serialize(trans, id, jeha) - return JobExportHistoryArchive.parse_obj(serialized_jeha) + return JobExportHistoryArchiveModel.parse_obj(serialized_jeha) else: # Valid request, just resource is not ready yet. trans.response.status = "202 Accepted" if jeha: serialized_jeha = self.history_export_view.serialize(trans, id, jeha) - return JobExportHistoryArchive.parse_obj(serialized_jeha) + return JobExportHistoryArchiveModel.parse_obj(serialized_jeha) else: assert job is not None, "logic error, don't have a jeha or a job" job_id = trans.security.encode_id(job.id) diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index 78d2e9e6149..e559cdab96d 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -891,7 +891,7 @@ class CreateHistoryPayload(Model): ) -class JobExportHistoryArchive(Model): +class JobExportHistoryArchiveModel(Model): id: EncodedDatabaseIdField = Field( ..., title="ID", @@ -934,6 +934,10 @@ class JobExportHistoryArchive(Model): ) +class JobExportHistoryArchiveCollection(Model): + __root__: List[JobExportHistoryArchiveModel] + + class LabelValuePair(BaseModel): """Generic Label/Value pair model.""" label: str = Field( From d0224f087c2fc67702049f1aec50a77df735d42d Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 26 Aug 2021 21:14:04 +0200 Subject: [PATCH 052/221] Adapt test to use JSON headers Required by FastAPI, see comment https://github.com/galaxyproject/galaxy/pull/12152#issuecomment-864134487 --- lib/galaxy_test/api/test_histories.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/lib/galaxy_test/api/test_histories.py b/lib/galaxy_test/api/test_histories.py index 48153bfc7f4..008a016c4b6 100644 --- a/lib/galaxy_test/api/test_histories.py +++ b/lib/galaxy_test/api/test_histories.py @@ -1,7 +1,6 @@ import time from requests import ( - post, put ) @@ -26,7 +25,7 @@ class BaseHistories: def _create_history(self, name): post_data = dict(name=name) - create_response = self._post("histories", data=post_data).json() + create_response = self._post("histories", data=post_data, json=True).json() self._assert_has_keys(create_response, "name", "id") self.assertEqual(create_response["name"], name) return create_response @@ -103,7 +102,7 @@ class HistoriesApiTestCase(ApiTestCase, BaseHistories): def test_purge(self): history_id = self._create_history("TestHistoryForPurge")["id"] data = {'purge': True} - self._delete(f"histories/{history_id}", data=data) + self._delete(f"histories/{history_id}", data=data, json=True) show_response = self._show(history_id) assert show_response["deleted"] assert show_response["purged"] @@ -186,23 +185,21 @@ class HistoriesApiTestCase(ApiTestCase, BaseHistories): def test_create_anonymous_fails(self): post_data = dict(name="CannotCreate") - # Using lower-level _api_url will cause key to not be injected. - histories_url = self._api_url("histories") - create_response = post(url=histories_url, data=post_data) + create_response = self._post("histories", data=post_data, anon=True, json=True) self._assert_status_code_is(create_response, 403) def test_create_without_session_fails(self): post_data = dict(name="SessionNeeded") # Using admin=True will boostrap an Admin user without session - create_response = self._post("histories", data=post_data, admin=True) + create_response = self._post("histories", data=post_data, admin=True, json=True) self._assert_status_code_is(create_response, 400) def test_create_tag(self): post_data = dict(name="TestHistoryForTag") - history_id = self._post("histories", data=post_data).json()["id"] + history_id = self._post("histories", data=post_data, json=True).json()["id"] tag_data = dict(value="awesometagvalue") tag_url = f"histories/{history_id}/tags/awesometagname" - tag_create_response = self._post(tag_url, data=tag_data) + tag_create_response = self._post(tag_url, data=tag_data, json=True) self._assert_status_code_is(tag_create_response, 200) # TODO: (CE) test_create_from_copy From 3815ed412ae86ef2433fd1b32a5afadd09f71dea Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 27 Aug 2021 12:56:29 +0200 Subject: [PATCH 053/221] Add FastAPI routes for histories API --- lib/galaxy/managers/histories.py | 35 ++- lib/galaxy/webapps/galaxy/api/histories.py | 279 ++++++++++++++++++++- 2 files changed, 293 insertions(+), 21 deletions(-) diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index b06b42b25ca..68743bf9ade 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -687,7 +687,7 @@ class HistoriesService(ServiceBase): def index( self, - trans, + trans: ProvidesHistoryContext, serialization_params: SerializationParams, filter_query_params: FilterQueryParams, deleted_only: Optional[bool] = False, @@ -817,7 +817,7 @@ class HistoriesService(ServiceBase): def show( self, - trans, + trans: ProvidesHistoryContext, serialization_params: SerializationParams, history_id: Optional[EncodedDatabaseIdField] = None, ): @@ -850,7 +850,7 @@ class HistoriesService(ServiceBase): def update( self, - trans, + trans: ProvidesHistoryContext, id: EncodedDatabaseIdField, payload, serialization_params: SerializationParams, @@ -879,7 +879,7 @@ class HistoriesService(ServiceBase): def delete( self, - trans, + trans: ProvidesHistoryContext, history_id: EncodedDatabaseIdField, serialization_params: SerializationParams, purge: bool = False, @@ -906,7 +906,7 @@ class HistoriesService(ServiceBase): def undelete( self, - trans, + trans: ProvidesHistoryContext, history_id: EncodedDatabaseIdField, serialization_params: SerializationParams, ): @@ -927,7 +927,7 @@ class HistoriesService(ServiceBase): def shared_with_me( self, - trans, + trans: ProvidesHistoryContext, serialization_params: SerializationParams, filter_query_params: FilterQueryParams, ): @@ -945,7 +945,7 @@ class HistoriesService(ServiceBase): def published( self, - trans, + trans: ProvidesHistoryContext, serialization_params: SerializationParams, filter_query_params: FilterQueryParams, ): @@ -961,7 +961,7 @@ class HistoriesService(ServiceBase): rval = [self._serialize_history(trans, history, serialization_params, default_view="summary") for history in histories] return rval - def citations(self, trans, history_id): + def citations(self, trans: ProvidesHistoryContext, history_id: EncodedDatabaseIdField): """ Return all the citations for the tools used to produce the datasets in the history. @@ -978,7 +978,7 @@ class HistoriesService(ServiceBase): tool_ids.add(tool_id) return [citation.to_dict("bibtex") for citation in self.citations_manager.citations_for_tool_ids(tool_ids)] - def index_exports(self, trans, id): + def index_exports(self, trans: ProvidesHistoryContext, id: EncodedDatabaseIdField): """ Get previous history exports (to links). Effectively returns serialized JEHA objects. @@ -1042,16 +1042,25 @@ class HistoriesService(ServiceBase): job_id = trans.security.encode_id(job.id) return JobIdResponse(job_id=job_id) - def archive_download(self, trans, id, jeha_id): + def archive_download( + self, + trans: ProvidesHistoryContext, + id: EncodedDatabaseIdField, + jeha_id: EncodedDatabaseIdField, + ): """ If ready and available, return raw contents of exported history. """ jeha = self.history_export_view.get_ready_jeha(trans, id, jeha_id) return self.manager.serve_ready_history_export(trans, jeha) - def get_custom_builds_metadata(self, trans, id: EncodedDatabaseIdField) -> CustomBuildsMetadataResponse: + def get_custom_builds_metadata( + self, + trans: ProvidesHistoryContext, + id: EncodedDatabaseIdField, + ) -> CustomBuildsMetadataResponse: """ - Returns meta data for custom builds. + Returns metadata for custom builds. """ history = self.manager.get_accessible(self.decode_id(id), trans.user, current_history=trans.history) installed_builds = [] @@ -1067,7 +1076,7 @@ class HistoriesService(ServiceBase): def _serialize_history( self, - trans, + trans: ProvidesHistoryContext, history: model.History, serialization_params: SerializationParams, default_view: str = "detailed", diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index 6b975db1445..572a5de237f 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -4,14 +4,24 @@ API operations on a history. .. seealso:: :class:`galaxy.model.History` """ import logging -from typing import Optional +from typing import ( + Any, + List, + Optional, + Union, +) from fastapi import ( Body, + Depends, Path, + Query, Response, status, ) +from pydantic.fields import Field +from pydantic.main import BaseModel +from starlette.responses import StreamingResponse from galaxy import ( util @@ -20,15 +30,29 @@ from galaxy.managers import ( histories, sharable, ) -from galaxy.managers.context import ProvidesUserContext -from galaxy.schema import FilterQueryParams +from galaxy.managers.context import ( + ProvidesHistoryContext, + ProvidesUserContext, +) +from galaxy.schema import ( + FilterQueryParams, + SerializationParams, +) from galaxy.schema.fields import ( EncodedDatabaseIdField, OrderParamField, ) from galaxy.schema.schema import ( CreateHistoryPayload, + CustomBuildsMetadataResponse, ExportHistoryArchivePayload, + HistoryBeta, + HistoryDetailed, + HistorySummary, + JobExportHistoryArchiveCollection, + JobExportHistoryArchiveModel, + JobIdResponse, + JobImportHistoryResponse, ) from galaxy.util import ( string_as_bool @@ -39,7 +63,10 @@ from galaxy.web import ( expose_api_anonymous_and_sessionless, expose_api_raw, ) -from galaxy.webapps.galaxy.api.configuration import parse_serialization_params +from galaxy.webapps.galaxy.api.common import ( + parse_serialization_params, + query_serialization_params, +) from . import ( BaseGalaxyAPIController, depends, @@ -51,17 +78,257 @@ log = logging.getLogger(__name__) router = Router(tags=['histories']) +AnyHistoryView = Union[HistoryBeta, HistoryDetailed, HistorySummary] + HistoryIdPathParam: EncodedDatabaseIdField = Path( ..., title="History ID", description="The encoded database identifier of the History." ) +JehaIDPathParam: EncodedDatabaseIdField = Path( + ..., + title='Job Export History ID', + description='The ID of the Job Export History Association.' +) + + +class HistoryFilterQueryParams(FilterQueryParams): + order: Optional[str] = OrderParamField(default_order="create_time-dsc") + + +class HistoryIndexParams(HistoryFilterQueryParams): + all: Optional[bool] = False + + +class DeleteHistoryPayload(BaseModel): + purge: bool = Field( + default=False, + title="Purge", + description="Whether to definitely remove this history from disk." + ) + @router.cbv class FastAPIHistories: service: histories.HistoriesService = depends(histories.HistoriesService) + @router.get( + '/api/histories', + summary='Returns histories for the current user.', + ) + def index( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + params: HistoryIndexParams = Depends(HistoryIndexParams), + serialization_params: SerializationParams = Depends(query_serialization_params), + deleted: bool = Query( # This is for backward compatibility but looks redundant + default=False, + title="Deleted Only", + description="Whether to return only deleted items.", + deprecated=True, # Marked as deprecated as it seems just like '/api/histories/deleted' + ) + ) -> List[AnyHistoryView]: + return self.service.index(trans, serialization_params, params, deleted_only=deleted, all_histories=params.all) + + @router.get( + '/api/histories/deleted', + summary='Returns deleted histories for the current user.', + ) + def index_deleted( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + params: HistoryIndexParams = Depends(HistoryIndexParams), + serialization_params: SerializationParams = Depends(query_serialization_params), + ) -> List[AnyHistoryView]: + return self.service.index(trans, serialization_params, params, deleted_only=True, all_histories=params.all) + + @router.get( + '/api/histories/published', + summary='Return all histories that are published.', + ) + def published( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + serialization_params: SerializationParams = Depends(query_serialization_params), + filter_params: HistoryFilterQueryParams = Depends(HistoryFilterQueryParams), + ) -> List[AnyHistoryView]: + return self.service.published(trans, serialization_params, filter_params) + + @router.get( + '/api/histories/shared_with_me', + summary='Return all histories that are shared with the current user.', + ) + def shared_with_me( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + serialization_params: SerializationParams = Depends(query_serialization_params), + filter_params: HistoryFilterQueryParams = Depends(HistoryFilterQueryParams), + ) -> List[AnyHistoryView]: + return self.service.shared_with_me(trans, serialization_params, filter_params) + + @router.get( + '/api/histories/most_recently_used', + summary='Returns the most recently used history of the user.', + ) + def show_recent( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + serialization_params: SerializationParams = Depends(query_serialization_params), + ) -> AnyHistoryView: + return self.service.show(trans, serialization_params) + + @router.get( + '/api/histories/{id}', + summary='Returns the history with the given ID.', + ) + def show( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + id: EncodedDatabaseIdField = HistoryIdPathParam, + serialization_params: SerializationParams = Depends(query_serialization_params), + ) -> AnyHistoryView: + return self.service.show(trans, serialization_params, id) + + @router.get( + '/api/histories/{id}/citations', + summary='Return all the citations for the tools used to produce the datasets in the history.', + ) + def citations( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + id: EncodedDatabaseIdField = HistoryIdPathParam, + ) -> List[Any]: + return self.service.citations(trans, id) + + @router.post( + '/api/histories', + summary='Returns the history with the given ID.', + ) + def create( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + payload: CreateHistoryPayload = Body(...), + serialization_params: SerializationParams = Depends(query_serialization_params), + ) -> Union[JobImportHistoryResponse, AnyHistoryView]: + return self.service.create(trans, payload, serialization_params) + + @router.delete( + '/api/histories/{id}', + summary='Marks the history with the given ID as deleted.', + ) + def delete( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + id: EncodedDatabaseIdField = HistoryIdPathParam, + serialization_params: SerializationParams = Depends(query_serialization_params), + purge: bool = Query(default=False), + payload: Optional[DeleteHistoryPayload] = Body(default=None) + ) -> AnyHistoryView: + if payload: + purge = payload.purge + return self.service.delete(trans, id, serialization_params, purge) + + @router.post( + '/api/histories/deleted/{id}/undelete', + summary="Restores a deleted history with the given ID (that hasn't been purged).", + ) + def undelete( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + id: EncodedDatabaseIdField = HistoryIdPathParam, + serialization_params: SerializationParams = Depends(query_serialization_params), + ) -> AnyHistoryView: + return self.service.undelete(trans, id, serialization_params) + + @router.put( + '/api/histories/{id}', + summary="Updates the values for the history with the given ID.", + ) + def update( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + id: EncodedDatabaseIdField = HistoryIdPathParam, + payload: Any = Body( + ..., + description="Object containing any of the editable fields of the history.", + ), + serialization_params: SerializationParams = Depends(query_serialization_params), + ) -> AnyHistoryView: + return self.service.update(trans, id, payload, serialization_params) + + @router.get( + '/api/histories/{id}/exports', + summary=( + "Get previous history exports (to links). Effectively returns serialized JEHA objects." + ), + ) + def index_exports( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + id: EncodedDatabaseIdField = HistoryIdPathParam, + ) -> JobExportHistoryArchiveCollection: + exports = self.service.index_exports(trans, id) + return JobExportHistoryArchiveCollection.parse_obj(exports) + + @router.put( # PUT instead of POST because multiple requests should just result in one object being created. + '/api/histories/{id}/exports', + summary=( + "Start job (if needed) to create history export for corresponding history." + ), + responses={ + 200: { + "description": "Object containing url to fetch export from.", + }, + 202: { + "description": "The exported archive file is not ready yet.", + } + }, + ) + def archive_export( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + id: EncodedDatabaseIdField = HistoryIdPathParam, + payload: ExportHistoryArchivePayload = Body(...), + ) -> Union[JobExportHistoryArchiveModel, JobIdResponse]: + return self.service.archive_export(trans, id, payload) + + @router.get( + '/api/histories/{id}/exports/{jeha_id}', + summary=( + "If ready and available, return raw contents of exported history. " + ), + response_class=StreamingResponse, + responses={ + 200: { + "description": "The archive file containing the History.", + } + }, + ) + def archive_download( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + id: EncodedDatabaseIdField = HistoryIdPathParam, + jeha_id: EncodedDatabaseIdField = JehaIDPathParam, + ): + """ + Use/poll ``PUT /api/histories/{id}/exports`` to initiate the creation + of such an export - when ready that route will return 200 status + code (instead of 202) with a JSON dictionary containing a ``download_url``. + """ + return self.service.archive_download(trans, id, jeha_id) + + @router.get( + '/api/histories/{id}/custom_builds_metadata', + summary="Returns meta data for custom builds.", + ) + def get_custom_builds_metadata( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + id: EncodedDatabaseIdField = HistoryIdPathParam, + ) -> CustomBuildsMetadataResponse: + return self.service.get_custom_builds_metadata(trans, id) + @router.get( '/api/histories/{id}/sharing', summary="Get the current sharing status of the given item.", @@ -151,10 +418,6 @@ class FastAPIHistories: return Response(status_code=status.HTTP_204_NO_CONTENT) -class HistoryFilterQueryParams(FilterQueryParams): - order: Optional[str] = OrderParamField(default_order="create_time-dsc") - - class HistoriesController(BaseGalaxyAPIController): service: histories.HistoriesService = depends(histories.HistoriesService) From c78a96288eaa682b3a84d1a4517cecdc169b07c0 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 27 Aug 2021 14:03:51 +0200 Subject: [PATCH 054/221] Move response status to API level The 204 status in `archive_export` should be changed at the API level and not the service level, especially while both, legacy and FastAPI controllers coexist as the response is handled differently. --- lib/galaxy/managers/histories.py | 11 +++++++++-- lib/galaxy/webapps/galaxy/api/histories.py | 17 +++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index 68743bf9ade..a6f291f2f7d 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -69,6 +69,8 @@ from galaxy.util import restore_text log = logging.getLogger(__name__) +HistoryArchiveExportResult = Union[JobExportHistoryArchiveModel, JobIdResponse] + class HDABasicInfo(BaseModel): id: EncodedDatabaseIdField @@ -990,7 +992,7 @@ class HistoriesService(ServiceBase): trans, id: EncodedDatabaseIdField, payload: ExportHistoryArchivePayload, - ) -> Union[JobExportHistoryArchiveModel, JobIdResponse]: + ) -> HistoryArchiveExportResult: """ start job (if needed) to create history export for corresponding history. @@ -1033,7 +1035,6 @@ class HistoriesService(ServiceBase): return JobExportHistoryArchiveModel.parse_obj(serialized_jeha) else: # Valid request, just resource is not ready yet. - trans.response.status = "202 Accepted" if jeha: serialized_jeha = self.history_export_view.serialize(trans, id, jeha) return JobExportHistoryArchiveModel.parse_obj(serialized_jeha) @@ -1042,6 +1043,12 @@ class HistoriesService(ServiceBase): job_id = trans.security.encode_id(job.id) return JobIdResponse(job_id=job_id) + def is_export_result_ready(self, export_result: HistoryArchiveExportResult) -> bool: + if isinstance(export_result, JobIdResponse): + return False + export_result = cast(JobExportHistoryArchiveModel, export_result) + return export_result.up_to_date and export_result.ready + def archive_download( self, trans: ProvidesHistoryContext, diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index 572a5de237f..ade1ad0b14c 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -50,8 +50,6 @@ from galaxy.schema.schema import ( HistoryDetailed, HistorySummary, JobExportHistoryArchiveCollection, - JobExportHistoryArchiveModel, - JobIdResponse, JobImportHistoryResponse, ) from galaxy.util import ( @@ -287,11 +285,15 @@ class FastAPIHistories: ) def archive_export( self, - trans: ProvidesHistoryContext = DependsOnTrans, + response: Response, + trans=DependsOnTrans, id: EncodedDatabaseIdField = HistoryIdPathParam, payload: ExportHistoryArchivePayload = Body(...), - ) -> Union[JobExportHistoryArchiveModel, JobIdResponse]: - return self.service.archive_export(trans, id, payload) + ) -> histories.HistoryArchiveExportResult: + export_result = self.service.archive_export(trans, id, payload) + if not self.service.is_export_result_ready(export_result): + response.status_code = status.HTTP_202_ACCEPTED + return export_result @router.get( '/api/histories/{id}/exports/{jeha_id}', @@ -720,7 +722,10 @@ class HistoriesController(BaseGalaxyAPIController): payload = payload or {} payload.update(kwds or {}) export_payload = ExportHistoryArchivePayload(**payload) - return self.service.archive_export(trans, id, export_payload) + export_result = self.service.archive_export(trans, id, export_payload) + if not self.service.is_export_result_ready(export_result): + trans.response.status = 202 + return export_result @expose_api_raw def archive_download(self, trans, id, jeha_id, **kwds): From 91eb9b0e6938c34add0000b57b1631eaf926bddb Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 30 Aug 2021 13:35:18 +0200 Subject: [PATCH 055/221] Handle `archive_download` action The implementation for the FastAPI version of the endpoint differs from the legacy one, so it cannot be totally reused like in the rest of the routes. The response is now treated as a Streaming response and the metadata is handled at the API level instead of the manager. --- lib/galaxy/managers/histories.py | 43 +++++++++++++++-- lib/galaxy/schema/types.py | 4 ++ lib/galaxy/webapps/galaxy/api/histories.py | 46 ++++++++++++++----- .../webapps/galaxy/controllers/history.py | 9 ---- 4 files changed, 79 insertions(+), 23 deletions(-) diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index a6f291f2f7d..40dd25c814e 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -63,6 +63,7 @@ from galaxy.schema.schema import ( JobImportHistoryResponse, LabelValuePair, ) +from galaxy.schema.types import LatestLiteral from galaxy.security.idencoding import IdEncodingHelper from galaxy.structured_app import MinimalManagerApp from galaxy.util import restore_text @@ -253,7 +254,8 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix job, _ = history_imp_tool.execute(trans, incoming=incoming) return job - def serve_ready_history_export(self, trans, jeha): + # TODO: remove this function when the legacy endpoint using it is removed + def legacy_serve_ready_history_export(self, trans, jeha): assert jeha.ready if jeha.compressed: trans.response.set_content_type('application/x-gzip') @@ -264,6 +266,16 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix archive = trans.app.object_store.get_filename(jeha.dataset) return open(archive, mode='rb') + def serve_ready_history_export(self, trans, jeha): + """ + Serves the history export archive for use as a streaming response so the file + doesn't need to be loaded into memory. + """ + assert jeha.ready + archive = trans.app.object_store.get_filename(jeha.dataset) + with open(archive, mode="rb") as archive_file: + yield from archive_file + def queue_history_export(self, trans, history, gzip=True, include_hidden=False, include_deleted=False, directory_uri=None, file_name=None): # Convert options to booleans. if isinstance(gzip, str): @@ -991,7 +1003,7 @@ class HistoriesService(ServiceBase): self, trans, id: EncodedDatabaseIdField, - payload: ExportHistoryArchivePayload, + payload: Optional[ExportHistoryArchivePayload] = None, ) -> HistoryArchiveExportResult: """ start job (if needed) to create history export for corresponding @@ -1003,6 +1015,8 @@ class HistoriesService(ServiceBase): :rtype: dict :returns: object containing url to fetch export from. """ + if payload is None: + payload = ExportHistoryArchivePayload() history = self.manager.get_accessible(self.decode_id(id), trans.user, current_history=trans.history) jeha = history.latest_export exporting_to_uri = payload.directory_uri @@ -1049,7 +1063,30 @@ class HistoriesService(ServiceBase): export_result = cast(JobExportHistoryArchiveModel, export_result) return export_result.up_to_date and export_result.ready + def get_ready_history_export( + self, + trans: ProvidesHistoryContext, + id: EncodedDatabaseIdField, + jeha_id: Union[EncodedDatabaseIdField, LatestLiteral], + ) -> model.JobExportHistoryArchive: + """Returns the exported history archive information if it's ready + or raises an exception if not.""" + return self.history_export_view.get_ready_jeha(trans, id, jeha_id) + def archive_download( + self, + trans: ProvidesHistoryContext, + jeha: model.JobExportHistoryArchive, + ): + """ + If ready and available, return raw contents of exported history + using a generator function. + """ + return self.manager.serve_ready_history_export(trans, jeha) + + # TODO: remove this function and HistoryManager.legacy_serve_ready_history_export when + # removing the legacy HistoriesController + def legacy_archive_download( self, trans: ProvidesHistoryContext, id: EncodedDatabaseIdField, @@ -1059,7 +1096,7 @@ class HistoriesService(ServiceBase): If ready and available, return raw contents of exported history. """ jeha = self.history_export_view.get_ready_jeha(trans, id, jeha_id) - return self.manager.serve_ready_history_export(trans, jeha) + return self.manager.legacy_serve_ready_history_export(trans, jeha) def get_custom_builds_metadata( self, diff --git a/lib/galaxy/schema/types.py b/lib/galaxy/schema/types.py index 803ab85f459..cf39c391d3a 100644 --- a/lib/galaxy/schema/types.py +++ b/lib/galaxy/schema/types.py @@ -1,3 +1,7 @@ +from typing_extensions import Literal + # Relative URLs cannot be validated with AnyUrl, they need a scheme. # Making them an alias of `str` for now RelativeUrl = str + +LatestLiteral = Literal["latest"] diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index ade1ad0b14c..953f83380c3 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -52,6 +52,7 @@ from galaxy.schema.schema import ( JobExportHistoryArchiveCollection, JobImportHistoryResponse, ) +from galaxy.schema.types import LatestLiteral from galaxy.util import ( string_as_bool ) @@ -84,10 +85,13 @@ HistoryIdPathParam: EncodedDatabaseIdField = Path( description="The encoded database identifier of the History." ) -JehaIDPathParam: EncodedDatabaseIdField = Path( - ..., +JehaIDPathParam: Union[EncodedDatabaseIdField, LatestLiteral] = Path( + default="latest", title='Job Export History ID', - description='The ID of the Job Export History Association.' + description=( + 'The ID of the specific Job Export History Association or ' + '`latest` (default) to download the last generated archive.' + ) ) @@ -288,8 +292,18 @@ class FastAPIHistories: response: Response, trans=DependsOnTrans, id: EncodedDatabaseIdField = HistoryIdPathParam, - payload: ExportHistoryArchivePayload = Body(...), + payload: Optional[ExportHistoryArchivePayload] = Body(None), ) -> histories.HistoryArchiveExportResult: + """This will start a job to create a history export archive. + + Calling this endpoint multiple times will return the 202 status code until the archive + has been completely generated and is ready to download. When ready, it will return + the 200 status code along with the download link information. + + If the history will be exported to a `directory_uri`, instead of returning the download + link information, the Job ID will be returned so it can be queried to determine when + the file has been written. + """ export_result = self.service.archive_export(trans, id, payload) if not self.service.is_export_result_ready(export_result): response.status_code = status.HTTP_202_ACCEPTED @@ -298,7 +312,7 @@ class FastAPIHistories: @router.get( '/api/histories/{id}/exports/{jeha_id}', summary=( - "If ready and available, return raw contents of exported history. " + "If ready and available, return raw contents of exported history as a downloadable archive." ), response_class=StreamingResponse, responses={ @@ -311,14 +325,22 @@ class FastAPIHistories: self, trans: ProvidesHistoryContext = DependsOnTrans, id: EncodedDatabaseIdField = HistoryIdPathParam, - jeha_id: EncodedDatabaseIdField = JehaIDPathParam, + jeha_id: Union[EncodedDatabaseIdField, LatestLiteral] = JehaIDPathParam, ): """ - Use/poll ``PUT /api/histories/{id}/exports`` to initiate the creation - of such an export - when ready that route will return 200 status - code (instead of 202) with a JSON dictionary containing a ``download_url``. + See ``PUT /api/histories/{id}/exports`` to initiate the creation + of the history export - when ready, that route will return 200 status + code (instead of 202) and this route can be used to download the archive. """ - return self.service.archive_download(trans, id, jeha_id) + jeha = self.service.get_ready_history_export(trans, id, jeha_id) + media_type = 'application/x-tar' + if jeha.compressed: + media_type = 'application/x-gzip' + return StreamingResponse( + self.service.archive_download(trans, jeha), + headers={'Content-Disposition': f'attachment; filename="{jeha.export_name}"'}, + media_type=media_type, + ) @router.get( '/api/histories/{id}/custom_builds_metadata', @@ -738,7 +760,9 @@ class HistoriesController(BaseGalaxyAPIController): code (instead of 202) with a JSON dictionary containing a ``download_url``. """ - return self.service.archive_download(trans, id, jeha_id) + # TODO: remove the HistoriesService.legacy_archive_download function when + # removing this endpoint + return self.service.legacy_archive_download(trans, id, jeha_id) @expose_api def get_custom_builds_metadata(self, trans, id, payload=None, **kwd): diff --git a/lib/galaxy/webapps/galaxy/controllers/history.py b/lib/galaxy/webapps/galaxy/controllers/history.py index 1e67db56ade..bcd9de57040 100644 --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -781,15 +781,6 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt return self.get_ave_item_rating_data(trans.sa_session, history) # TODO: used in display_base.mako - @web.expose - def export_archive(self, trans, id=None, jeha_id="latest"): - """ Export a history to an archive. """ - # - # Get history to export. - # - jeha = self.history_export_view.get_ready_jeha(trans, id, jeha_id) - return self.history_manager.serve_ready_history_export(trans, jeha) - @web.expose @web.json @web.require_login("get history name and link") From 4948a12c07ae2a6aeedab5945b5eb7e15ae14eb8 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 31 Aug 2021 18:45:39 +0200 Subject: [PATCH 056/221] Handle `create` action using form data The create action involves file uploading for history import, so we have to use Form data instead of JSON. See docs https://fastapi.tiangolo.com/tutorial/request-forms-and-files/ - Add required `python-multipart` dependency - Add decorator `as_form` that allows to use a pydantic model as form-data - Handle how FastAPI deals with uploaded files using SpooledTemporaryFile --- lib/galaxy/dependencies/dev-requirements.txt | 1 + .../dependencies/pinned-requirements.txt | 1 + lib/galaxy/managers/histories.py | 20 ++++++++++++- lib/galaxy/schema/schema.py | 4 +-- lib/galaxy/webapps/galaxy/api/__init__.py | 30 +++++++++++++++++++ lib/galaxy/webapps/galaxy/api/histories.py | 10 +++++-- lib/galaxy_test/api/test_histories.py | 2 +- pyproject.toml | 1 + 8 files changed, 62 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index b9b7d50f7d0..96cb077bb58 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -174,6 +174,7 @@ pytest-html==3.1.1; python_version >= "3.6" pytest-json-report==1.2.4 pytest-metadata==1.11.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" pytest-mock==3.6.0; python_version >= "3.6" +python-multipart==0.0.5 pytest-postgresql==2.6.1; python_version >= "3.6" pytest-pythonpath==0.7.3 pytest-shard==0.1.2; python_version >= "3.6" diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 9e179bad992..e84afa9c896 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -152,6 +152,7 @@ pysam==0.16.0.1 python-dateutil==2.8.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.3.0" python-jose==3.2.0 python-keystoneclient==4.1.1; python_version >= "3.6" +python-multipart==0.0.5 python-neutronclient==7.2.1; python_version >= "3.6" python-novaclient==17.2.1; python_version >= "3.6" python-swiftclient==3.10.1 diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index 40dd25c814e..78ff437e65c 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -7,6 +7,12 @@ created (or copied) by users over the course of an analysis. import glob import logging import os +import shutil +from pathlib import Path +from tempfile import ( + NamedTemporaryFile, + SpooledTemporaryFile, +) from typing import ( cast, List, @@ -791,7 +797,7 @@ class HistoriesService(ServiceBase): copy_this_history_id = payload.history_id all_datasets = payload.all_datasets - if payload.archive_source is not None: + if payload.archive_source is not None or hasattr(payload.archive_file, "file"): archive_source = payload.archive_source archive_file = payload.archive_file if archive_source: @@ -799,6 +805,8 @@ class HistoriesService(ServiceBase): elif archive_file is not None and hasattr(archive_file, "file"): archive_source = archive_file.file.name archive_type = HistoryImportArchiveSourceType.file + if isinstance(archive_file.file, SpooledTemporaryFile): + archive_source = self._save_upload_file_tmp(archive_file) else: raise glx_exceptions.MessageException("Please provide a url or file.") job = self.manager.queue_history_import(trans, archive_type=archive_type, archive_source=archive_source) @@ -829,6 +837,16 @@ class HistoriesService(ServiceBase): return self._serialize_history(trans, new_history, serialization_params) + def _save_upload_file_tmp(self, upload_file) -> str: + try: + suffix = Path(upload_file.filename).suffix + with NamedTemporaryFile(delete=False, suffix=suffix) as tmp: + shutil.copyfileobj(upload_file.file, tmp) + tmp_path = Path(tmp.name) + finally: + upload_file.file.close() + return str(tmp_path) + def show( self, trans: ProvidesHistoryContext, diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index e559cdab96d..b79dede176b 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -875,8 +875,6 @@ class CreateHistoryPayload(Model): title="Archive Source", description=( "The URL that will generate the archive to import when `archive_type='url'`. " - # This seems a bit odd but the create history action expects `archive_source` to be != None - "When importing from a file using `archive_file`, please set `archive_source=''`." ), ) archive_type: Optional[HistoryImportArchiveSourceType] = Field( @@ -887,7 +885,7 @@ class CreateHistoryPayload(Model): archive_file: Optional[Any] = Field( default=None, title="Archive File", - description="Detailed file information when importing the history from a file.", + description="Uploaded file information when importing the history from a file.", ) diff --git a/lib/galaxy/webapps/galaxy/api/__init__.py b/lib/galaxy/webapps/galaxy/api/__init__.py index 12549e7d865..5ed571f6253 100644 --- a/lib/galaxy/webapps/galaxy/api/__init__.py +++ b/lib/galaxy/webapps/galaxy/api/__init__.py @@ -1,6 +1,7 @@ """ This module *does not* contain API routes. It exclusively contains dependencies to be used in FastAPI routes """ +import inspect from typing import ( Any, AsyncGenerator, @@ -12,12 +13,14 @@ from typing import ( from fastapi import ( Cookie, + Form, Header, Query, ) from fastapi.params import Depends from fastapi_utils.cbv import cbv from fastapi_utils.inferring_router import InferringRouter +from pydantic.main import BaseModel try: from starlette_context import context as request_context except ImportError: @@ -201,3 +204,30 @@ class Router(InferringRouter): https://fastapi-utils.davidmontague.xyz/user-guide/class-based-views/ """ return cbv(self) + + +def as_form(cls: Type[BaseModel]): + """ + Adds an as_form class method to decorated models. The as_form class method + can be used with FastAPI endpoints. + + See https://github.com/tiangolo/fastapi/issues/2387#issuecomment-731662551 + """ + new_params = [ + inspect.Parameter( + field.alias, + inspect.Parameter.POSITIONAL_ONLY, + default=(Form(field.default) if not field.required else Form(...)), + ) + for field in cls.__fields__.values() + ] + + async def _as_form(**data): + return cls(**data) + + sig = inspect.signature(_as_form) + sig = sig.replace(parameters=new_params) + _as_form.__signature__ = sig # type: ignore + # setattr(cls, "as_form", _as_form) + cls.as_form = _as_form + return cls diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index 953f83380c3..6ec3d8727ee 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -67,6 +67,7 @@ from galaxy.webapps.galaxy.api.common import ( query_serialization_params, ) from . import ( + as_form, BaseGalaxyAPIController, depends, DependsOnTrans, @@ -111,6 +112,11 @@ class DeleteHistoryPayload(BaseModel): ) +@as_form +class CreateHistoryFormData(CreateHistoryPayload): + """Uses Form data instead of JSON""" + + @router.cbv class FastAPIHistories: service: histories.HistoriesService = depends(histories.HistoriesService) @@ -207,10 +213,10 @@ class FastAPIHistories: '/api/histories', summary='Returns the history with the given ID.', ) - def create( + async def create( self, trans: ProvidesHistoryContext = DependsOnTrans, - payload: CreateHistoryPayload = Body(...), + payload: CreateHistoryPayload = Depends(CreateHistoryFormData.as_form), serialization_params: SerializationParams = Depends(query_serialization_params), ) -> Union[JobImportHistoryResponse, AnyHistoryView]: return self.service.create(trans, payload, serialization_params) diff --git a/lib/galaxy_test/api/test_histories.py b/lib/galaxy_test/api/test_histories.py index 008a016c4b6..8220751bf94 100644 --- a/lib/galaxy_test/api/test_histories.py +++ b/lib/galaxy_test/api/test_histories.py @@ -25,7 +25,7 @@ class BaseHistories: def _create_history(self, name): post_data = dict(name=name) - create_response = self._post("histories", data=post_data, json=True).json() + create_response = self._post("histories", data=post_data).json() self._assert_has_keys(create_response, "name", "id") self.assertEqual(create_response["name"], name) return create_response diff --git a/pyproject.toml b/pyproject.toml index 40b0927a01a..be2365f53fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,7 @@ pykwalify = "*" pyparsing = "*" pysam = "*" python = "^3.6" +python-multipart = "*" pyuwsgi = "*" PyYAML = "*" refgenconf = ">=0.12.0" From 1de73222e2393fc8a027379c124dd1537ca51ee5 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 31 Aug 2021 19:01:48 +0200 Subject: [PATCH 057/221] Adapt test_parse_serialization_params to expect a model --- test/unit/api/test_configuration.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/unit/api/test_configuration.py b/test/unit/api/test_configuration.py index cb86640b5cb..c1fe35e0c55 100644 --- a/test/unit/api/test_configuration.py +++ b/test/unit/api/test_configuration.py @@ -5,15 +5,15 @@ def test_parse_serialization_params(): view, default_view = 'a', 'b' keys = 'foo' serialized = parse_serialization_params(view, keys, default_view) - assert serialized['view'] == view - assert serialized['default_view'] == default_view - assert serialized['keys'] == [keys] + assert serialized.view == view + assert serialized.default_view == default_view + assert serialized.keys == [keys] keys = 'foo,bar,baz' serialized = parse_serialization_params(view, keys, default_view) - assert serialized['keys'] == ['foo', 'bar', 'baz'] + assert serialized.keys == ['foo', 'bar', 'baz'] serialized = parse_serialization_params(default_view=default_view) - assert serialized['view'] is None - assert serialized['keys'] is None - assert serialized['default_view'] == default_view + assert serialized.view is None + assert serialized.keys is None + assert serialized.default_view == default_view From f0692a651eefa8a421490af8a11c8657abcdb2a7 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 31 Aug 2021 19:03:53 +0200 Subject: [PATCH 058/221] Fix mypy --- lib/galaxy/webapps/galaxy/api/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/__init__.py b/lib/galaxy/webapps/galaxy/api/__init__.py index 5ed571f6253..6746744b9e4 100644 --- a/lib/galaxy/webapps/galaxy/api/__init__.py +++ b/lib/galaxy/webapps/galaxy/api/__init__.py @@ -228,6 +228,5 @@ def as_form(cls: Type[BaseModel]): sig = inspect.signature(_as_form) sig = sig.replace(parameters=new_params) _as_form.__signature__ = sig # type: ignore - # setattr(cls, "as_form", _as_form) - cls.as_form = _as_form + cls.as_form = _as_form # type: ignore return cls From 0ec958cbfcb3c7fb6fbc56d89a68dc497365a621 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 1 Sep 2021 12:43:35 +0200 Subject: [PATCH 059/221] Fix double dict conversion in configuration API This was unintentionally introduced in 15e69de380a01ed8bef6b1c9d12dbbf6b042d5ae --- lib/galaxy/webapps/galaxy/api/configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/configuration.py b/lib/galaxy/webapps/galaxy/api/configuration.py index 4c1d9c9eb73..439a31f35e2 100644 --- a/lib/galaxy/webapps/galaxy/api/configuration.py +++ b/lib/galaxy/webapps/galaxy/api/configuration.py @@ -209,6 +209,6 @@ def _user_to_model(user, security): return UserModel(**user.to_dict(view='element', value_mapper={'id': security.encode_id})) if user else None -def _index(manager, trans, view, keys): +def _index(manager: ConfigurationManager, trans, view, keys): serialization_params = parse_serialization_params(view, keys, 'all') - return manager.get_configuration(trans, serialization_params.dict()) + return manager.get_configuration(trans, serialization_params) From 5abf6863bcbf1120483f35beeb189cacc585cf90 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 1 Sep 2021 15:17:16 +0200 Subject: [PATCH 060/221] Generate API URLs for downloading history exports This may not be the best way to generate the URLs since this is forcing all to be API URLs but I couldn't think of a better solution to make it work for both legacy and FastAPI frameworks. --- lib/galaxy/managers/histories.py | 8 +++----- lib/galaxy/webapps/galaxy/api/histories.py | 1 + 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index 78ff437e65c..8f3c6b90c79 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -386,11 +386,9 @@ class HistoryExportView: def serialize(self, trans, history_id, jeha): rval = jeha.to_dict() encoded_jeha_id = trans.security.encode_id(jeha.id) - api_url = self.app.url_for("history_archive_download", id=history_id, jeha_id=encoded_jeha_id) - # this URL is less likely to be blocked by a proxy and require an API key, so export - # older-style controller version for use with within the GUI and such. - external_url = self.app.url_for(controller='history', action="export_archive", id=history_id, qualified=True) - external_permanent_url = self.app.url_for(controller='history', action="export_archive", id=history_id, jeha_id=encoded_jeha_id, qualified=True) + api_url = trans.url_builder("history_archive_download", id=history_id, jeha_id=encoded_jeha_id) + external_url = trans.url_builder("history_archive_download", id=history_id, jeha_id="latest", qualified=True) + external_permanent_url = trans.url_builder("history_archive_download", id=history_id, jeha_id=encoded_jeha_id, qualified=True) rval["download_url"] = api_url rval["external_download_latest_url"] = external_url rval["external_download_permanent_url"] = external_permanent_url diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index 6ec3d8727ee..b7297ea0448 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -317,6 +317,7 @@ class FastAPIHistories: @router.get( '/api/histories/{id}/exports/{jeha_id}', + name="history_archive_download", summary=( "If ready and available, return raw contents of exported history as a downloadable archive." ), From 6ceb9d79da8d7fd25161acbb04d35eb40c4f55e7 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 2 Sep 2021 12:12:17 +0200 Subject: [PATCH 061/221] Fix `run_as` to be passed in the headers The legacy API framework allowed to pass the `run_as` user in the payload but, moving forward to the new framework, looks like the headers is a better place for it (see https://github.com/galaxyproject/galaxy/pull/11342/commits/91e7d5191d70f3b286fdf86b39af657bede35f09). - Adapt existing framework tests to pass in the `run_as` user in the headers instead of the payload. - Fix the legacy framework not retrieving the `run_as` from the headers when the payload was not in JSON (i.e. Form data) - Add a bit more of documentation for the OpenAPI schema. --- lib/galaxy/web/framework/decorators.py | 6 +++--- lib/galaxy/webapps/galaxy/api/__init__.py | 10 +++++++++- lib/galaxy_test/api/test_framework.py | 22 ++++++++++++++++------ 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/web/framework/decorators.py b/lib/galaxy/web/framework/decorators.py index bd8c261a4c7..91de832d71d 100644 --- a/lib/galaxy/web/framework/decorators.py +++ b/lib/galaxy/web/framework/decorators.py @@ -221,9 +221,9 @@ def __extract_payload_from_request(trans, func, kwargs): # should ideally be in reverse, with the if clause being a check for application/json and the else clause assuming a standard encoding # such as multipart/form-data. Leaving it as is for backward compatibility, just in case. payload = loads(unicodify(trans.request.body)) - run_as = trans.request.headers.get('run-as') - if run_as: - payload['run_as'] = run_as + run_as = trans.request.headers.get('run-as') + if run_as: + payload['run_as'] = run_as return payload diff --git a/lib/galaxy/webapps/galaxy/api/__init__.py b/lib/galaxy/webapps/galaxy/api/__init__.py index 6746744b9e4..8b568e112ea 100644 --- a/lib/galaxy/webapps/galaxy/api/__init__.py +++ b/lib/galaxy/webapps/galaxy/api/__init__.py @@ -104,7 +104,15 @@ def get_api_user( user_manager: UserManager = depends(UserManager), key: Optional[str] = Query(None), x_api_key: Optional[str] = Header(None), - run_as: Optional[EncodedDatabaseIdField] = Header(None, title='Run as User', description='Admins and ')) -> Optional[User]: + run_as: Optional[EncodedDatabaseIdField] = Header( + default=None, + title='Run as User', + description=( + 'The user ID that will be used to effectively make this API call. ' + 'Only admins and designated users can make API calls on behalf of other users.' + ) + ) +) -> Optional[User]: api_key = key or x_api_key if not api_key: return None diff --git a/lib/galaxy_test/api/test_framework.py b/lib/galaxy_test/api/test_framework.py index dae920b9433..3bc3f51ec0b 100644 --- a/lib/galaxy_test/api/test_framework.py +++ b/lib/galaxy_test/api/test_framework.py @@ -11,21 +11,31 @@ class ApiFrameworkTestCase(ApiTestCase): # Next several tests test the API's run_as functionality. def test_user_cannont_run_as(self): - post_data = dict(name="TestHistory1", run_as="another_user") + run_as_user = self._setup_user("for_run_as@bx.psu.edu") + post_data = dict(name="TestHistory1") # Normal user cannot run_as... - create_response = self._post("histories", data=post_data) + create_response = self._post( + "histories", data=post_data, + headers={'run-as': run_as_user["id"]}, + ) self._assert_status_code_is(create_response, 403) def test_run_as_invalid_user(self): - post_data = dict(name="TestHistory1", run_as="another_user") + post_data = dict(name="TestHistory1") # admin user can run_as, but this user doesn't exist, expect 400. - create_response = self._post("histories", data=post_data, admin=True) + create_response = self._post( + "histories", data=post_data, + headers={'run-as': "another_user"}, admin=True, + ) self._assert_status_code_is(create_response, 400) def test_run_as_valid_user(self): run_as_user = self._setup_user("for_run_as@bx.psu.edu") - post_data = dict(name="TestHistory1", run_as=run_as_user["id"]) + post_data = dict(name="TestHistory1") # Use run_as with admin user and for another user just created, this # should work. - create_response = self._post("histories", data=post_data, admin=True) + create_response = self._post( + "histories", data=post_data, + headers={'run-as': run_as_user["id"]}, admin=True, + ) self._assert_status_code_is(create_response, 200) From bded1164db0f7049da854206c2b37cd70cccb411 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 2 Sep 2021 16:47:41 +0200 Subject: [PATCH 062/221] Fix `current_history` not set in `SessionRequestContext` --- lib/galaxy/webapps/galaxy/api/__init__.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/__init__.py b/lib/galaxy/webapps/galaxy/api/__init__.py index 8b568e112ea..cd59b7d9cf0 100644 --- a/lib/galaxy/webapps/galaxy/api/__init__.py +++ b/lib/galaxy/webapps/galaxy/api/__init__.py @@ -150,11 +150,22 @@ class UrlBuilder: DependsOnUser = Depends(get_user) +def get_current_history_from_session(galaxy_session: Optional[model.GalaxySession]) -> Optional[model.History]: + if galaxy_session: + return galaxy_session.current_history + return None + + def get_trans(request: Request, app: StructuredApp = DependsOnApp, user: Optional[User] = Depends(get_user), galaxy_session: Optional[model.GalaxySession] = Depends(get_session), ) -> SessionRequestContext: url_builder = UrlBuilder(request) - return SessionRequestContext(app=app, user=user, galaxy_session=galaxy_session, url_builder=url_builder, host=request.client.host) + return SessionRequestContext( + app=app, user=user, + galaxy_session=galaxy_session, + url_builder=url_builder, host=request.client.host, + history=get_current_history_from_session(galaxy_session), + ) DependsOnTrans = Depends(get_trans) From c1ff44c7e6c97ae59778e5be08df899e4832d321 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 2 Sep 2021 16:59:42 +0200 Subject: [PATCH 063/221] Add `Any` as a valid model for History view This is the only way I can think of to support arbitrary serialization `keys` being requested. Otherwise, the validation will fail because the required fields in the annotated models may not be present in the resulting `custom` serialized history. --- lib/galaxy/managers/histories.py | 6 ++-- lib/galaxy/schema/schema.py | 8 +++++ lib/galaxy/webapps/galaxy/api/histories.py | 40 ++++++++++------------ 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index 8f3c6b90c79..1636b185594 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -57,13 +57,11 @@ from galaxy.schema import ( ) from galaxy.schema.fields import EncodedDatabaseIdField from galaxy.schema.schema import ( + AnyHistoryView, CreateHistoryPayload, CustomBuildsMetadataResponse, ExportHistoryArchivePayload, - HistoryBeta, - HistoryDetailed, HistoryImportArchiveSourceType, - HistorySummary, JobExportHistoryArchiveModel, JobIdResponse, JobImportHistoryResponse, @@ -1140,7 +1138,7 @@ class HistoriesService(ServiceBase): history: model.History, serialization_params: SerializationParams, default_view: str = "detailed", - ) -> Union[HistoryBeta, HistoryDetailed, HistorySummary]: + ) -> Union[AnyHistoryView]: """ Returns a dictionary with the corresponding values depending on the serialization parameters provided. diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index b79dede176b..a34415bd0a9 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -811,6 +811,14 @@ class HistoryBeta(HistoryDetailed): ) +AnyHistoryView = Union[ + HistoryBeta, HistoryDetailed, HistorySummary, + # Any will cover those cases in which only specific `keys` are requested + # otherwise the validation will fail because the required fields are not returned + Any, +] + + class ExportHistoryArchivePayload(Model): gzip: Optional[bool] = Field( default=True, diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index b7297ea0448..ab5649eb941 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -43,12 +43,10 @@ from galaxy.schema.fields import ( OrderParamField, ) from galaxy.schema.schema import ( + AnyHistoryView, CreateHistoryPayload, CustomBuildsMetadataResponse, ExportHistoryArchivePayload, - HistoryBeta, - HistoryDetailed, - HistorySummary, JobExportHistoryArchiveCollection, JobImportHistoryResponse, ) @@ -78,9 +76,7 @@ log = logging.getLogger(__name__) router = Router(tags=['histories']) -AnyHistoryView = Union[HistoryBeta, HistoryDetailed, HistorySummary] - -HistoryIdPathParam: EncodedDatabaseIdField = Path( +HistoryIDPathParam: EncodedDatabaseIdField = Path( ..., title="History ID", description="The encoded database identifier of the History." @@ -193,7 +189,7 @@ class FastAPIHistories: def show( self, trans: ProvidesHistoryContext = DependsOnTrans, - id: EncodedDatabaseIdField = HistoryIdPathParam, + id: EncodedDatabaseIdField = HistoryIDPathParam, serialization_params: SerializationParams = Depends(query_serialization_params), ) -> AnyHistoryView: return self.service.show(trans, serialization_params, id) @@ -205,7 +201,7 @@ class FastAPIHistories: def citations( self, trans: ProvidesHistoryContext = DependsOnTrans, - id: EncodedDatabaseIdField = HistoryIdPathParam, + id: EncodedDatabaseIdField = HistoryIDPathParam, ) -> List[Any]: return self.service.citations(trans, id) @@ -228,7 +224,7 @@ class FastAPIHistories: def delete( self, trans: ProvidesHistoryContext = DependsOnTrans, - id: EncodedDatabaseIdField = HistoryIdPathParam, + id: EncodedDatabaseIdField = HistoryIDPathParam, serialization_params: SerializationParams = Depends(query_serialization_params), purge: bool = Query(default=False), payload: Optional[DeleteHistoryPayload] = Body(default=None) @@ -244,7 +240,7 @@ class FastAPIHistories: def undelete( self, trans: ProvidesHistoryContext = DependsOnTrans, - id: EncodedDatabaseIdField = HistoryIdPathParam, + id: EncodedDatabaseIdField = HistoryIDPathParam, serialization_params: SerializationParams = Depends(query_serialization_params), ) -> AnyHistoryView: return self.service.undelete(trans, id, serialization_params) @@ -256,7 +252,7 @@ class FastAPIHistories: def update( self, trans: ProvidesHistoryContext = DependsOnTrans, - id: EncodedDatabaseIdField = HistoryIdPathParam, + id: EncodedDatabaseIdField = HistoryIDPathParam, payload: Any = Body( ..., description="Object containing any of the editable fields of the history.", @@ -274,7 +270,7 @@ class FastAPIHistories: def index_exports( self, trans: ProvidesHistoryContext = DependsOnTrans, - id: EncodedDatabaseIdField = HistoryIdPathParam, + id: EncodedDatabaseIdField = HistoryIDPathParam, ) -> JobExportHistoryArchiveCollection: exports = self.service.index_exports(trans, id) return JobExportHistoryArchiveCollection.parse_obj(exports) @@ -297,7 +293,7 @@ class FastAPIHistories: self, response: Response, trans=DependsOnTrans, - id: EncodedDatabaseIdField = HistoryIdPathParam, + id: EncodedDatabaseIdField = HistoryIDPathParam, payload: Optional[ExportHistoryArchivePayload] = Body(None), ) -> histories.HistoryArchiveExportResult: """This will start a job to create a history export archive. @@ -331,7 +327,7 @@ class FastAPIHistories: def archive_download( self, trans: ProvidesHistoryContext = DependsOnTrans, - id: EncodedDatabaseIdField = HistoryIdPathParam, + id: EncodedDatabaseIdField = HistoryIDPathParam, jeha_id: Union[EncodedDatabaseIdField, LatestLiteral] = JehaIDPathParam, ): """ @@ -356,7 +352,7 @@ class FastAPIHistories: def get_custom_builds_metadata( self, trans: ProvidesHistoryContext = DependsOnTrans, - id: EncodedDatabaseIdField = HistoryIdPathParam, + id: EncodedDatabaseIdField = HistoryIDPathParam, ) -> CustomBuildsMetadataResponse: return self.service.get_custom_builds_metadata(trans, id) @@ -367,7 +363,7 @@ class FastAPIHistories: def sharing( self, trans: ProvidesUserContext = DependsOnTrans, - id: EncodedDatabaseIdField = HistoryIdPathParam, + id: EncodedDatabaseIdField = HistoryIDPathParam, ) -> sharable.SharingStatus: """Return the sharing status of the item.""" return self.service.shareable_service.sharing(trans, id) @@ -379,7 +375,7 @@ class FastAPIHistories: def enable_link_access( self, trans: ProvidesUserContext = DependsOnTrans, - id: EncodedDatabaseIdField = HistoryIdPathParam, + id: EncodedDatabaseIdField = HistoryIDPathParam, ) -> sharable.SharingStatus: """Makes this item accessible by a URL link and return the current sharing status.""" return self.service.shareable_service.enable_link_access(trans, id) @@ -391,7 +387,7 @@ class FastAPIHistories: def disable_link_access( self, trans: ProvidesUserContext = DependsOnTrans, - id: EncodedDatabaseIdField = HistoryIdPathParam, + id: EncodedDatabaseIdField = HistoryIDPathParam, ) -> sharable.SharingStatus: """Makes this item inaccessible by a URL link and return the current sharing status.""" return self.service.shareable_service.disable_link_access(trans, id) @@ -403,7 +399,7 @@ class FastAPIHistories: def publish( self, trans: ProvidesUserContext = DependsOnTrans, - id: EncodedDatabaseIdField = HistoryIdPathParam, + id: EncodedDatabaseIdField = HistoryIDPathParam, ) -> sharable.SharingStatus: """Makes this item publicly available by a URL link and return the current sharing status.""" return self.service.shareable_service.publish(trans, id) @@ -415,7 +411,7 @@ class FastAPIHistories: def unpublish( self, trans: ProvidesUserContext = DependsOnTrans, - id: EncodedDatabaseIdField = HistoryIdPathParam, + id: EncodedDatabaseIdField = HistoryIDPathParam, ) -> sharable.SharingStatus: """Removes this item from the published list and return the current sharing status.""" return self.service.shareable_service.unpublish(trans, id) @@ -427,7 +423,7 @@ class FastAPIHistories: def share_with_users( self, trans: ProvidesUserContext = DependsOnTrans, - id: EncodedDatabaseIdField = HistoryIdPathParam, + id: EncodedDatabaseIdField = HistoryIDPathParam, payload: sharable.ShareWithPayload = Body(...) ) -> sharable.ShareWithStatus: """Shares this item with specific users and return the current sharing status.""" @@ -441,7 +437,7 @@ class FastAPIHistories: def set_slug( self, trans: ProvidesUserContext = DependsOnTrans, - id: EncodedDatabaseIdField = HistoryIdPathParam, + id: EncodedDatabaseIdField = HistoryIDPathParam, payload: sharable.SetSlugPayload = Body(...), ): """Sets a new slug to access this item by URL. The new slug must be unique.""" From a5d87ffc4aa037a10d3359ae3171cf5d47b68442 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 8 Sep 2021 16:44:57 +0200 Subject: [PATCH 064/221] Add support for JSON payload in `create` operation + test --- lib/galaxy/webapps/galaxy/api/histories.py | 13 ++++++++++++- lib/galaxy_test/api/test_histories.py | 8 ++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index ab5649eb941..9489944c301 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -16,6 +16,7 @@ from fastapi import ( Depends, Path, Query, + Request, Response, status, ) @@ -207,14 +208,24 @@ class FastAPIHistories: @router.post( '/api/histories', - summary='Returns the history with the given ID.', + summary='Creates a new history.', ) async def create( self, + request: Request, trans: ProvidesHistoryContext = DependsOnTrans, payload: CreateHistoryPayload = Depends(CreateHistoryFormData.as_form), serialization_params: SerializationParams = Depends(query_serialization_params), ) -> Union[JobImportHistoryResponse, AnyHistoryView]: + """The new history can also be copied form a existing history or imported from an archive or URL.""" + # This action needs to work both with json and x-www-form-urlencoded payloads. + # The way to support different content types on the same path operation is reading + # the request directly and parse it depending on the content type. + # We will assume x-www-form-urlencoded by default to deal with possible file uploads. + # See https://github.com/tiangolo/fastapi/issues/990#issuecomment-639615888 + if "application/json" in request.headers.get("content-type", ""): + body = await request.json() + payload = CreateHistoryPayload.parse_obj(body) return self.service.create(trans, payload, serialization_params) @router.delete( diff --git a/lib/galaxy_test/api/test_histories.py b/lib/galaxy_test/api/test_histories.py index 8220751bf94..e9306a9d417 100644 --- a/lib/galaxy_test/api/test_histories.py +++ b/lib/galaxy_test/api/test_histories.py @@ -43,6 +43,14 @@ class HistoriesApiTestCase(ApiTestCase, BaseHistories): indexed_history = [h for h in index_response if h["id"] == created_id][0] self.assertEqual(indexed_history["name"], "TestHistory1") + def test_create_history_json(self): + name = "TestHistoryJson" + post_data = dict(name=name) + create_response = self._post("histories", data=post_data, json=True).json() + self._assert_has_keys(create_response, "name", "id") + self.assertEqual(create_response["name"], name) + return create_response + def test_show_history(self): history_id = self._create_history("TestHistoryForShow")["id"] show_response = self._show(history_id) From 36469a6ad3fb76a3ad6e9902b0e9b9fcdb201d6d Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 9 Sep 2021 17:09:34 +0200 Subject: [PATCH 065/221] Fix 204 response status condition on `archive_export` --- lib/galaxy/managers/histories.py | 19 +++++++------------ lib/galaxy/webapps/galaxy/api/histories.py | 8 ++++---- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index 1636b185594..9c37241bfd4 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -14,7 +14,6 @@ from tempfile import ( SpooledTemporaryFile, ) from typing import ( - cast, List, Optional, Set, @@ -1018,7 +1017,7 @@ class HistoriesService(ServiceBase): trans, id: EncodedDatabaseIdField, payload: Optional[ExportHistoryArchivePayload] = None, - ) -> HistoryArchiveExportResult: + ) -> Tuple[HistoryArchiveExportResult, bool]: """ start job (if needed) to create history export for corresponding history. @@ -1051,31 +1050,27 @@ class HistoriesService(ServiceBase): else: job = jeha.job + ready = bool((up_to_date and jeha.ready) or exporting_to_uri) + if exporting_to_uri: # we don't have a jeha, there will never be a download_url. Just let # the client poll on the created job_id to determine when the file has been # written. job_id = trans.security.encode_id(job.id) - return JobIdResponse(job_id=job_id) + return (JobIdResponse(job_id=job_id), ready) if up_to_date and jeha.ready: serialized_jeha = self.history_export_view.serialize(trans, id, jeha) - return JobExportHistoryArchiveModel.parse_obj(serialized_jeha) + return (JobExportHistoryArchiveModel.parse_obj(serialized_jeha), ready) else: # Valid request, just resource is not ready yet. if jeha: serialized_jeha = self.history_export_view.serialize(trans, id, jeha) - return JobExportHistoryArchiveModel.parse_obj(serialized_jeha) + return (JobExportHistoryArchiveModel.parse_obj(serialized_jeha), ready) else: assert job is not None, "logic error, don't have a jeha or a job" job_id = trans.security.encode_id(job.id) - return JobIdResponse(job_id=job_id) - - def is_export_result_ready(self, export_result: HistoryArchiveExportResult) -> bool: - if isinstance(export_result, JobIdResponse): - return False - export_result = cast(JobExportHistoryArchiveModel, export_result) - return export_result.up_to_date and export_result.ready + return (JobIdResponse(job_id=job_id), ready) def get_ready_history_export( self, diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index 9489944c301..d3ff2eb548e 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -317,8 +317,8 @@ class FastAPIHistories: link information, the Job ID will be returned so it can be queried to determine when the file has been written. """ - export_result = self.service.archive_export(trans, id, payload) - if not self.service.is_export_result_ready(export_result): + export_result, ready = self.service.archive_export(trans, id, payload) + if not ready: response.status_code = status.HTTP_202_ACCEPTED return export_result @@ -758,8 +758,8 @@ class HistoriesController(BaseGalaxyAPIController): payload = payload or {} payload.update(kwds or {}) export_payload = ExportHistoryArchivePayload(**payload) - export_result = self.service.archive_export(trans, id, export_payload) - if not self.service.is_export_result_ready(export_result): + export_result, ready = self.service.archive_export(trans, id, export_payload) + if not ready: trans.response.status = 202 return export_result From 8a7f94f9a25f4e4ea37081c2007e13ffd61f5152 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 10 Sep 2021 10:15:54 +0200 Subject: [PATCH 066/221] Make create path operation sync We need to use `await request.json()` because that function returns a coroutine, but we can move this into an async path dependency as Marius suggested so the path operation remains sync. --- lib/galaxy/managers/histories.py | 1 + lib/galaxy/webapps/galaxy/api/__init__.py | 8 ++++++++ lib/galaxy/webapps/galaxy/api/histories.py | 14 +++++++------- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index 9c37241bfd4..b9a0aa7304f 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -14,6 +14,7 @@ from tempfile import ( SpooledTemporaryFile, ) from typing import ( + cast, List, Optional, Set, diff --git a/lib/galaxy/webapps/galaxy/api/__init__.py b/lib/galaxy/webapps/galaxy/api/__init__.py index cd59b7d9cf0..33ff2bf809d 100644 --- a/lib/galaxy/webapps/galaxy/api/__init__.py +++ b/lib/galaxy/webapps/galaxy/api/__init__.py @@ -249,3 +249,11 @@ def as_form(cls: Type[BaseModel]): _as_form.__signature__ = sig # type: ignore cls.as_form = _as_form # type: ignore return cls + + +async def try_get_request_body_as_json(request: Request) -> Optional[Any]: + """Returns the request body as a JSON object if the content type is JSON.""" + if "application/json" in request.headers.get("content-type", ""): + body = await request.json() + return body + return None diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index d3ff2eb548e..cf75d14ddf0 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -16,7 +16,6 @@ from fastapi import ( Depends, Path, Query, - Request, Response, status, ) @@ -71,6 +70,7 @@ from . import ( depends, DependsOnTrans, Router, + try_get_request_body_as_json ) log = logging.getLogger(__name__) @@ -210,22 +210,22 @@ class FastAPIHistories: '/api/histories', summary='Creates a new history.', ) - async def create( + def create( self, - request: Request, trans: ProvidesHistoryContext = DependsOnTrans, payload: CreateHistoryPayload = Depends(CreateHistoryFormData.as_form), + payload_as_json: Optional[Any] = Depends(try_get_request_body_as_json), serialization_params: SerializationParams = Depends(query_serialization_params), ) -> Union[JobImportHistoryResponse, AnyHistoryView]: """The new history can also be copied form a existing history or imported from an archive or URL.""" # This action needs to work both with json and x-www-form-urlencoded payloads. # The way to support different content types on the same path operation is reading # the request directly and parse it depending on the content type. - # We will assume x-www-form-urlencoded by default to deal with possible file uploads. + # We will assume x-www-form-urlencoded (payload) by default to deal with possible file uploads + # and if the content type is explicitly JSON, we will use payload_as_json instead. # See https://github.com/tiangolo/fastapi/issues/990#issuecomment-639615888 - if "application/json" in request.headers.get("content-type", ""): - body = await request.json() - payload = CreateHistoryPayload.parse_obj(body) + if payload_as_json: + payload = CreateHistoryPayload.parse_obj(payload_as_json) return self.service.create(trans, payload, serialization_params) @router.delete( From 6d7b39e732d69be85d54db0d6e37de87d783a37c Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 13 Sep 2021 10:09:55 +0200 Subject: [PATCH 067/221] Remove redundant Union --- lib/galaxy/managers/histories.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index b9a0aa7304f..454dadb2b5e 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -1134,7 +1134,7 @@ class HistoriesService(ServiceBase): history: model.History, serialization_params: SerializationParams, default_view: str = "detailed", - ) -> Union[AnyHistoryView]: + ) -> AnyHistoryView: """ Returns a dictionary with the corresponding values depending on the serialization parameters provided. From d4a71e7194da1441e3a3b95d8ec24f6aa7cb838e Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 13 Sep 2021 18:19:21 +0200 Subject: [PATCH 068/221] Use FileResponse instead of StreamingResponse --- lib/galaxy/managers/histories.py | 18 +++++++++++------- lib/galaxy/webapps/galaxy/api/histories.py | 18 +++++++++--------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index 454dadb2b5e..90bdad98264 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -270,15 +270,13 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix archive = trans.app.object_store.get_filename(jeha.dataset) return open(archive, mode='rb') - def serve_ready_history_export(self, trans, jeha): + def get_ready_history_export_file_path(self, trans, jeha) -> str: """ Serves the history export archive for use as a streaming response so the file doesn't need to be loaded into memory. """ assert jeha.ready - archive = trans.app.object_store.get_filename(jeha.dataset) - with open(archive, mode="rb") as archive_file: - yield from archive_file + return trans.app.object_store.get_filename(jeha.dataset) def queue_history_export(self, trans, history, gzip=True, include_hidden=False, include_deleted=False, directory_uri=None, file_name=None): # Convert options to booleans. @@ -1083,16 +1081,22 @@ class HistoriesService(ServiceBase): or raises an exception if not.""" return self.history_export_view.get_ready_jeha(trans, id, jeha_id) - def archive_download( + def get_archive_download_path( self, trans: ProvidesHistoryContext, jeha: model.JobExportHistoryArchive, - ): + ) -> str: """ If ready and available, return raw contents of exported history using a generator function. """ - return self.manager.serve_ready_history_export(trans, jeha) + return self.manager.get_ready_history_export_file_path(trans, jeha) + + def get_archive_media_type(self, jeha: model.JobExportHistoryArchive): + media_type = 'application/x-tar' + if jeha.compressed: + media_type = 'application/x-gzip' + return media_type # TODO: remove this function and HistoryManager.legacy_serve_ready_history_export when # removing the legacy HistoriesController diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index cf75d14ddf0..8ca6f9db301 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -21,7 +21,7 @@ from fastapi import ( ) from pydantic.fields import Field from pydantic.main import BaseModel -from starlette.responses import StreamingResponse +from starlette.responses import FileResponse from galaxy import ( util @@ -89,7 +89,8 @@ JehaIDPathParam: Union[EncodedDatabaseIdField, LatestLiteral] = Path( description=( 'The ID of the specific Job Export History Association or ' '`latest` (default) to download the last generated archive.' - ) + ), + example="latest" ) @@ -328,7 +329,7 @@ class FastAPIHistories: summary=( "If ready and available, return raw contents of exported history as a downloadable archive." ), - response_class=StreamingResponse, + response_class=FileResponse, responses={ 200: { "description": "The archive file containing the History.", @@ -347,13 +348,12 @@ class FastAPIHistories: code (instead of 202) and this route can be used to download the archive. """ jeha = self.service.get_ready_history_export(trans, id, jeha_id) - media_type = 'application/x-tar' - if jeha.compressed: - media_type = 'application/x-gzip' - return StreamingResponse( - self.service.archive_download(trans, jeha), - headers={'Content-Disposition': f'attachment; filename="{jeha.export_name}"'}, + media_type = self.service.get_archive_media_type(jeha) + file_path = self.service.get_archive_download_path(trans, jeha) + return FileResponse( + path=file_path, media_type=media_type, + filename=jeha.export_name, ) @router.get( From 9c9f8dc6fd54a2768f8dec50799c40c12e4c9efa Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 21 Sep 2021 15:01:39 +0200 Subject: [PATCH 069/221] Drop leftover error logging There's no error here, I assume that was used for debugging, but it's very verbose on startup. --- lib/galaxy/tools/parameters/validation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/tools/parameters/validation.py b/lib/galaxy/tools/parameters/validation.py index 3b5e322e0d9..103abe62cbd 100644 --- a/lib/galaxy/tools/parameters/validation.py +++ b/lib/galaxy/tools/parameters/validation.py @@ -190,7 +190,6 @@ class ExpressionValidator(Validator): message = f"Value '%s' does not evaluate to {'True' if negate == 'false' else 'False'} for '{expression}'" super().__init__(message, negate) # Save compiled expression, code objects are thread safe (right?) - log.error(f"ExpressionValidator expression {expression}") self.expression = compile(expression, '', 'eval') def validate(self, value, trans=None): From a6f07ea9a8c7f9c7fe4123d2c77d110391d4c21e Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Mon, 20 Sep 2021 21:53:07 -0400 Subject: [PATCH 070/221] Excise remaining worker code --- .../History/caching/CacheWorker.worker.js | 37 ------ .../src/components/History/caching/index.js | 32 ----- .../History/caching/loadDscContent.js | 2 +- .../History/caching/loadHistoryContents.js | 2 +- .../History/caching/workerClient.js | 119 ------------------ .../History/caching/workerConfig.js | 20 --- 6 files changed, 2 insertions(+), 210 deletions(-) delete mode 100644 client/src/components/History/caching/CacheWorker.worker.js delete mode 100644 client/src/components/History/caching/workerClient.js delete mode 100644 client/src/components/History/caching/workerConfig.js diff --git a/client/src/components/History/caching/CacheWorker.worker.js b/client/src/components/History/caching/CacheWorker.worker.js deleted file mode 100644 index b23768e08dd..00000000000 --- a/client/src/components/History/caching/CacheWorker.worker.js +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Instantiates worker with with externally defined api - */ - -// TODO: isn't @babel/polyfill bad now? -import "@babel/polyfill"; -import { expose } from "threads/worker"; -import { asObservable } from "./asObservable"; -import { configure } from "./workerConfig"; -import * as api from "./CacheApi"; - -const { - monitorContentQuery, - monitorDscQuery, - monitorHistoryContent, - monitorCollectionContent, - loadHistoryContents, - loadDscContent, - pollHistory, - ...promises -} = api; - -expose({ - configure, - - // observable operators - monitorContentQuery: asObservable(monitorContentQuery), - monitorDscQuery: asObservable(monitorDscQuery), - monitorHistoryContent: asObservable(monitorHistoryContent), - monitorCollectionContent: asObservable(monitorCollectionContent), - loadHistoryContents: asObservable(loadHistoryContents), - loadDscContent: asObservable(loadDscContent), - pollHistory: asObservable(pollHistory), - - // promise functions - ...promises, -}); diff --git a/client/src/components/History/caching/index.js b/client/src/components/History/caching/index.js index 1dd1c91f6aa..b913e6f3ba0 100644 --- a/client/src/components/History/caching/index.js +++ b/client/src/components/History/caching/index.js @@ -16,35 +16,3 @@ export { } from "./CacheApi"; export { wipeDatabase, clearHistoryDateStore } from "./CacheApi"; - -// TODO: The above exports bypass the worker completely for now, swap back to below to use. -//import { toPromise, toOperator } from "./workerClient"; -///** -// * monitor cache for changes -// */ -//export const monitorContentQuery = toOperator("monitorContentQuery"); -//export const monitorDscQuery = toOperator("monitorDscQuery"); -//export const monitorHistoryContent = toOperator("monitorHistoryContent"); -//export const monitorCollectionContent = toOperator("monitorCollectionContent"); -// -///** -// * Loaders -// */ -//export const loadHistoryContents = toOperator("loadHistoryContents"); -//export const loadDscContent = toOperator("loadDscContent"); -// -///** -// * Cache promise functions -// */ -//export const cacheContent = toPromise("cacheContent"); -//export const getCachedContent = toPromise("getCachedContent"); -//export const uncacheContent = toPromise("uncacheContent"); -//export const bulkCacheContent = toPromise("bulkCacheContent"); -//export const cacheCollectionContent = toPromise("cacheCollectionContent"); -//export const getCachedCollectionContent = toPromise("getCachedCollectionContent"); -//export const bulkCacheDscContent = toPromise("bulkCacheDscContent"); -//export const getContentByTypeId = toPromise("getContentByTypeId"); -// -//// Debugging -//export const wipeDatabase = toPromise("wipeDatabase"); -//export const clearHistoryDateStore = toPromise("clearHistoryDateStore"); diff --git a/client/src/components/History/caching/loadDscContent.js b/client/src/components/History/caching/loadDscContent.js index aa65fd96092..4d47a3d8d95 100644 --- a/client/src/components/History/caching/loadDscContent.js +++ b/client/src/components/History/caching/loadDscContent.js @@ -8,7 +8,7 @@ import { nth } from "utils/observable"; import { requestWithUpdateTime } from "./operators/requestWithUpdateTime"; import { bulkCacheDscContent } from "./db"; import { SearchParams } from "../model/SearchParams"; -import { prependPath } from "./workerConfig"; +import { prependPath } from "utils/redirect"; import { summarizeCacheOperation, dateStore } from "./loadHistoryContents"; import { show } from "utils/observable"; diff --git a/client/src/components/History/caching/loadHistoryContents.js b/client/src/components/History/caching/loadHistoryContents.js index 0941297c382..0727b6fb901 100644 --- a/client/src/components/History/caching/loadHistoryContents.js +++ b/client/src/components/History/caching/loadHistoryContents.js @@ -3,7 +3,7 @@ import { map, pluck, share, filter } from "rxjs/operators"; import { hydrate } from "utils/observable"; import { areDefined } from "utils/validation"; import { requestWithUpdateTime } from "./operators/requestWithUpdateTime"; -import { prependPath } from "./workerConfig"; +import { prependPath } from "utils/redirect"; import { bulkCacheContent } from "./db"; import { SearchParams } from "../model/SearchParams"; import { createDateStore } from "../model/DateStore"; diff --git a/client/src/components/History/caching/workerClient.js b/client/src/components/History/caching/workerClient.js deleted file mode 100644 index c8f16bb2a80..00000000000 --- a/client/src/components/History/caching/workerClient.js +++ /dev/null @@ -1,119 +0,0 @@ -import { defer, from, of, pipe } from "rxjs"; -import { filter, finalize, materialize, map, mergeMap, shareReplay, mergeAll } from "rxjs/operators"; -import { v4 as uuidv4 } from "uuid"; -import { spawn } from "threads"; -import CacheWorker from "./CacheWorker.worker.js"; -import config from "config"; -import { getRootFromIndexLink } from "onload/getRootFromIndexLink"; - -/** - * @constant Observable yields the worker thread instance - */ -// prettier-ignore -const threadInstance$ = defer(() => of(config).pipe( - mergeMap(buildThread) -)); - -const thread$ = threadInstance$.pipe(shareReplay(1)); - -const buildThread = async (cfg) => { - const thread = await spawn(new CacheWorker()); - if (!thread) { - throw new MissingWorkerError(); - } - - // Configure the worker This is sending in settings that are derived from - // galaxy's absurd global application instance or written directly to - // the document, which will not be available in the worker. - const root = getRootFromIndexLink(); - const workerConfigs = { ...cfg, root }; - await thread.configure(workerConfigs); - - return thread; -}; - -// glorified pluck operator -// prettier-ignore -const method = (fnName) => pipe( - map((thread) => { - if (!(fnName in thread)) { - throw new MissingWorkerMethodError(fnName); - } - return thread[fnName]; - }) -); - -/** - * Give this a string of the function name on the worker thread instance, - * returns an observable operator that transparently calls a matching oprator - * from inside the worker . - * - * @param {string} fnName Name of an exposed property on the thread object - */ -// prettier-ignore -export const toOperator = (fnName) => { - const method$ = thread$.pipe(method(fnName)); - - // Result of the returned method call will be an "ObservablePromise", a - // custom object returned by thread library that the author probably thought - // was clever. We need to fix that by turning it back to a real observable - - const cleanMethod$ = method$.pipe( - map((f) => (...args) => from(f(...args))) // I'm a real boy now! - ); - - const operator = (cfg = {}) => (src$) => { - // identifies subscription so we can match external observable with - // itnernal observable - const id = uuidv4(); - - return cleanMethod$.pipe( - mergeMap((method) => src$.pipe( - materialize(), - map((notification) => method({ id, cfg, fnName, ...notification })), - // first emission will be the observable created by threads - // that's the only one we want, rest should be nulls - filter(Boolean), - // subscribe to the observable threads made - mergeAll(), - // unsub when exterior observable completes - finalize(() => method({ id, kind: "C" })) - )) - ); - }; - - return operator; -}; - -/** - * Returns an async function from a property on the thread object. This is - * actually what was already there, but we're using thread$ to manage the - * lifetime of the worker instance, so we'll derive the function from the thread - * observable. - * - * @param {string} workerMethod Name of method inside the worker - * @return {Function} Function that returns a promise - */ -export const toPromise = (fnName) => { - return async (...request) => { - const methodPromise = thread$.pipe(method(fnName)).toPromise(); - const fn = await methodPromise; - return await fn(...request); - }; -}; - -/** - * Custom Errors - */ - -class MissingWorkerError extends Error {} - -class MissingWorkerMethodError extends Error { - constructor(missingMethod, ...args) { - const msg = ` - Missing method on client cache worker: ${missingMethod}. - Please write a function named ${missingMethod} in caching/cacheWorker.js - `; - super(msg, ...args); - } -} diff --git a/client/src/components/History/caching/workerConfig.js b/client/src/components/History/caching/workerConfig.js deleted file mode 100644 index da0eff22b3c..00000000000 --- a/client/src/components/History/caching/workerConfig.js +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Configuration var for inside the worker, must be set when worker - * is fired up because we can't reach the document and that's how galaxy - * backbone code gets some of its config. - */ -export const workerConfig = { root: "/" }; - -export const configure = (options = {}) => { - Object.assign(workerConfig, options); -}; - -/** - * Prepend against this config. Can't access document so we can't use - * the standard one from utils - */ -const slashCleanup = /(\/)+/g; -export function prependPath(path) { - const root = workerConfig.root; - return `${root}/${path}`.replace(slashCleanup, "/"); -} From 62d26500438aabb4eeaa07afc7a50150105482f7 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 21 Sep 2021 09:32:58 -0400 Subject: [PATCH 071/221] Remove workflow API hooks for tool shed. We removed all the tool shed and GUI side stuff for this a while ago (#8978), it is worth simplifying the workflow API while we're at it. If we merge this - I'll work on improving the docstrings here to reflect the actual options. --- lib/galaxy/webapps/galaxy/api/workflows.py | 26 ++++------------------ 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index a67153f0e9d..3f2582b7bd6 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -268,14 +268,10 @@ class WorkflowsAPIController(BaseGalaxyAPIController, UsesStoredWorkflowMixin, U .. tip:: When executing a workflow externally (e.g. from a script) it is recommended to use the :func:`galaxy.webapps.galaxy.api.workflows.WorkflowsAPIController.invoke` method below instead. - If installed_repository_file or from_history_id is specified a new - workflow will be created for this user. Otherwise, workflow_id must be - specified and this API method will cause a workflow to execute. + If from_history_id is specified a new workflow will be created for this user. Otherwise, + workflow_id must be specified and this API method will cause a workflow to execute. - :param installed_repository_file The path of a workflow to import. Either workflow_id, installed_repository_file or from_history_id must be specified - :type installed_repository_file str - - :param workflow_id: An existing workflow id. Either workflow_id, installed_repository_file or from_history_id must be specified + :param workflow_id: An existing workflow id. Either workflow_id or from_history_id must be specified :type workflow_id: str :param parameters: If workflow_id is set - see _step_parameters() in lib/galaxy/workflow/run_request.py @@ -293,7 +289,7 @@ class WorkflowsAPIController(BaseGalaxyAPIController, UsesStoredWorkflowMixin, U :param replacement_params: If workflow_id is set - an optional dictionary used when renaming datasets :type replacement_params: dict - :param from_history_id: Id of history to extract a workflow from. Either workflow_id, installed_repository_file or from_history_id must be specified + :param from_history_id: Id of history to extract a workflow from. Either workflow_id or from_history_id must be specified :type from_history_id: str :param job_ids: If from_history_id is set - optional list of jobs to include when extracting a workflow from history @@ -318,7 +314,6 @@ class WorkflowsAPIController(BaseGalaxyAPIController, UsesStoredWorkflowMixin, U ways_to_create = { 'archive_source', 'workflow_id', - 'installed_repository_file', 'from_history_id', 'from_path', 'shared_workflow_id', @@ -333,19 +328,6 @@ class WorkflowsAPIController(BaseGalaxyAPIController, UsesStoredWorkflowMixin, U message = f"Only one parameter among - {', '.join(ways_to_create)} - must be specified" raise exceptions.RequestParameterInvalidException(message) - if 'installed_repository_file' in payload: - if not trans.user_is_admin: - raise exceptions.AdminRequiredException() - installed_repository_file = payload.get('installed_repository_file', '') - if not os.path.exists(installed_repository_file): - raise exceptions.RequestParameterInvalidException(f"Workflow file '{installed_repository_file}' not found") - elif os.path.getsize(os.path.abspath(installed_repository_file)) > 0: - with open(installed_repository_file, encoding='utf-8') as f: - workflow_data = f.read() - return self.__api_import_from_archive(trans, workflow_data, payload=payload) - else: - raise exceptions.MessageException("You attempted to open an empty file.") - if 'archive_source' in payload: archive_source = payload['archive_source'] archive_file = payload.get('archive_file') From 8f8ff23f318f8deaaaec0e4dc88a714f132d44df Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 21 Sep 2021 10:02:06 -0400 Subject: [PATCH 072/221] Test cases to verify only admins can import workflows from paths. --- lib/galaxy_test/api/test_workflows.py | 17 +++++++++++++++++ lib/galaxy_test/base/populators.py | 6 +++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index 5fe51e492c0..12d3fde3331 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -1,4 +1,5 @@ import json +import os import time from json import dumps from uuid import uuid4 @@ -4353,6 +4354,22 @@ input_c: unpublished_worklow = self._put(f'workflows/{workflow_id}', data={'published': False}, json=True).json() assert not unpublished_worklow['published'] + def test_workflow_from_path_requires_admin(self): + # There are two ways to import workflows from paths, just verify both require an admin. + workflow_directory = self._test_driver.mkdtemp() + workflow_path = os.path.join(workflow_directory, "workflow.yml") + with open(workflow_path, "w") as f: + f.write(WORKFLOW_NESTED_REPLACEMENT_PARAMETER) + import_response = self.workflow_populator.import_workflow_from_path_raw(workflow_path) + self._assert_status_code_is(import_response, 403) + self._assert_error_code_is(import_response, error_codes.ADMIN_REQUIRED) + + path_as_uri = f"file://{workflow_path}" + import_data = dict(archive_source=path_as_uri) + import_response = self._post("workflows", data=import_data) + self._assert_status_code_is(import_response, 403) + self._assert_error_code_is(import_response, error_codes.ADMIN_REQUIRED) + def _invoke_paused_workflow(self, history_id): workflow = self.workflow_populator.load_workflow_from_resource("test_workflow_pause") workflow_id = self.workflow_populator.create_workflow(workflow) diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index 55ddd106f2c..5f393a71c42 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -847,11 +847,15 @@ class BaseWorkflowPopulator(BasePopulator): workflow = self.load_workflow(name) return self.create_workflow(workflow, **create_kwds) - def import_workflow_from_path(self, from_path: str) -> str: + def import_workflow_from_path_raw(self, from_path: str) -> Response: data = dict( from_path=from_path ) import_response = self._post("workflows", data=data) + return import_response + + def import_workflow_from_path(self, from_path: str) -> str: + import_response = self.import_workflow_from_path_raw(from_path) api_asserts.assert_status_code_is(import_response, 200) return import_response.json()["id"] From 137d2ae1f4c50bb9bc735a036727da7f2f24b8fb Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 20 Sep 2021 11:20:47 +0200 Subject: [PATCH 073/221] more linting for requirements - check for missing version - check for missing name see https://github.com/galaxyproject/tools-iuc/pull/3978#discussion_r712919797 --- lib/galaxy/tool_util/linters/general.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/linters/general.py b/lib/galaxy/tool_util/linters/general.py index 6b08d2ed3ab..d4fa44f0856 100644 --- a/lib/galaxy/tool_util/linters/general.py +++ b/lib/galaxy/tool_util/linters/general.py @@ -63,8 +63,12 @@ def lint_general(tool_source, lint_ctx): requirements, containers = tool_source.parse_requirements_and_containers() for r in requirements: + if r.name == '': + lint_ctx.error("Requirement without name found") + if r.version is None or r.version == '': + lint_ctx.warn(f"Requirement {r.name} defines no version") # Warn requirement attributes with leading/trailing whitespace: - if r.version != r.version.strip(): + elif r.version != r.version.strip(): lint_ctx.warn( WARN_WHITESPACE_MSG % ('Requirement version', r.version)) From 4c22af41ec70c0d6f03c0d2a1a4fdd3d8fe47f16 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 21 Sep 2021 15:51:34 +0200 Subject: [PATCH 074/221] add test and restructure tests in multiple files --- .../tool_util/test_tool_linters_general.py | 87 +++++++++++++++++++ ..._linters.py => test_tool_linters_input.py} | 80 +---------------- .../tool_util/test_tool_linters_outputs.py | 63 ++++++++++++++ 3 files changed, 151 insertions(+), 79 deletions(-) create mode 100644 test/unit/tool_util/test_tool_linters_general.py rename test/unit/tool_util/{test_tool_linters.py => test_tool_linters_input.py} (71%) create mode 100644 test/unit/tool_util/test_tool_linters_outputs.py diff --git a/test/unit/tool_util/test_tool_linters_general.py b/test/unit/tool_util/test_tool_linters_general.py new file mode 100644 index 00000000000..7d5284b625b --- /dev/null +++ b/test/unit/tool_util/test_tool_linters_general.py @@ -0,0 +1,87 @@ +import pytest + +from galaxy.tool_util.lint import LintContext +from galaxy.tool_util.linters import general +from galaxy.tool_util.parser.xml import XmlToolSource +from galaxy.util import etree + + +def lint_general(xml_tree, lint_ctx): + """Wrap calling of lint_general to provide XmlToolSource argument. + + This allows general.lint_general to be called with the other linters which + take an XmlTree as an argument. + """ + tool_source = XmlToolSource(xml_tree) + return general.lint_general(tool_source, lint_ctx) + + +WHITESPACE_IN_VERSIONS_AND_NAMES = """ + +""" + +REQUIREMENT_WO_VERSION = """ + +""" + +TESTS = [ + ( + WHITESPACE_IN_VERSIONS_AND_NAMES, lint_general, + lambda x: + "Tool version contains whitespace, this may cause errors: [ 1.0.1 ]." in x.warn_messages + and "Tool name contains whitespace, this may cause errors: [ BWA Mapper ]." in x.warn_messages + and "Requirement version contains whitespace, this may cause errors: [ 1.2.5 ]." in x.warn_messages + and "Tool ID contains whitespace - this is discouraged: [bwa tool]." + and len(x.warn_messages) == 4 and len(x.error_messages) == 0 + ), + ( + REQUIREMENT_WO_VERSION, lint_general, + lambda x: + "Requirement bwa defines no version" in x.warn_messages + and "Requirement without name found" in x.error_messages + and len(x.warn_messages) == 1 and len(x.error_messages) == 1 + ), +] + +TEST_IDS = [ + 'hazardous whitespace', + 'requirement without version', +] + + +@pytest.mark.parametrize('tool_xml,lint_func,assert_func', TESTS, ids=TEST_IDS) +def test_tool_xml(tool_xml, lint_func, assert_func): + lint_ctx = LintContext('all') + tree = etree.ElementTree(element=etree.fromstring(tool_xml)) + lint_ctx.lint(name="test_lint", lint_func=lint_func, lint_target=tree) + assert assert_func(lint_ctx), ( + f"Warnings: {lint_ctx.warn_messages}\n" + f"Errors: {lint_ctx.error_messages}" + ) diff --git a/test/unit/tool_util/test_tool_linters.py b/test/unit/tool_util/test_tool_linters_input.py similarity index 71% rename from test/unit/tool_util/test_tool_linters.py rename to test/unit/tool_util/test_tool_linters_input.py index 0dab32dba2e..6b21f69abb4 100644 --- a/test/unit/tool_util/test_tool_linters.py +++ b/test/unit/tool_util/test_tool_linters_input.py @@ -1,21 +1,10 @@ import pytest from galaxy.tool_util.lint import LintContext -from galaxy.tool_util.linters import general, inputs, outputs -from galaxy.tool_util.parser.xml import XmlToolSource +from galaxy.tool_util.linters import inputs from galaxy.util import etree -def lint_general(xml_tree, lint_ctx): - """Wrap calling of lint_general to provide XmlToolSource argument. - - This allows general.lint_general to be called with the other linters which - take an XmlTree as an argument. - """ - tool_source = XmlToolSource(xml_tree) - return general.lint_general(tool_source, lint_ctx) - - NO_SECTIONS_XML = """ """ -WHITESPACE_IN_VERSIONS_AND_NAMES = """ - -""" - VALIDATOR_INCOMPATIBILITIES = """ """ -# check that linter accepts format source for collection elements as means to specify format -# and that the linter warns if format and format_source are used -OUTPUTS_COLLECTION_FORMAT_SOURCE = """ - -""" - -# check that linter does not complain about missing format if from_tool_provided_metadata is used -OUTPUTS_DISCOVER_TOOL_PROVIDED_METADATA = """ - -""" TESTS = [ ( NO_SECTIONS_XML, inputs.lint_inputs, @@ -210,15 +155,6 @@ TESTS = [ and "Select parameter [select_fd_fdt] options uses 'from_dataset' and 'from_data_table' attribute." in x.error_messages and len(x.warn_messages) == 0 and len(x.error_messages) == 5 ), - ( - WHITESPACE_IN_VERSIONS_AND_NAMES, lint_general, - lambda x: - "Tool version contains whitespace, this may cause errors: [ 1.0.1 ]." in x.warn_messages - and "Tool name contains whitespace, this may cause errors: [ BWA Mapper ]." in x.warn_messages - and "Requirement version contains whitespace, this may cause errors: [ 1.2.5 ]." in x.warn_messages - and "Tool ID contains whitespace - this is discouraged: [bwa tool]." - and len(x.warn_messages) == 4 and len(x.error_messages) == 0 - ), ( VALIDATOR_INCOMPATIBILITIES, inputs.lint_inputs, lambda x: @@ -229,17 +165,6 @@ TESTS = [ and "Parameter [param_name]: 'regex' validators need to define an 'expression' attribute" in x.error_messages and len(x.warn_messages) == 1 and len(x.error_messages) == 4 ), - ( - OUTPUTS_COLLECTION_FORMAT_SOURCE, outputs.lint_output, - lambda x: - "Tool data output reverse should use either format_source or format/ext" in x.warn_messages - and len(x.warn_messages) == 1 and len(x.error_messages) == 0 - ), - ( - OUTPUTS_DISCOVER_TOOL_PROVIDED_METADATA, outputs.lint_output, - lambda x: - len(x.warn_messages) == 0 and len(x.error_messages) == 0 - ), ] TEST_IDS = [ @@ -249,10 +174,7 @@ TEST_IDS = [ 'select duplicated options', 'select deprecations', 'select option definitions', - 'hazardous whitespace', 'validator imcompatibilities', - 'outputs collection static elements with format_source', - 'outputs discover datatsets with tool provided metadata' ] diff --git a/test/unit/tool_util/test_tool_linters_outputs.py b/test/unit/tool_util/test_tool_linters_outputs.py new file mode 100644 index 00000000000..5737f39789a --- /dev/null +++ b/test/unit/tool_util/test_tool_linters_outputs.py @@ -0,0 +1,63 @@ +import pytest + +from galaxy.tool_util.lint import LintContext +from galaxy.tool_util.linters import outputs +from galaxy.util import etree + + +# check that linter accepts format source for collection elements as means to specify format +# and that the linter warns if format and format_source are used +OUTPUTS_COLLECTION_FORMAT_SOURCE = """ + +""" + +# check that linter does not complain about missing format if from_tool_provided_metadata is used +OUTPUTS_DISCOVER_TOOL_PROVIDED_METADATA = """ + +""" +TESTS = [ + ( + OUTPUTS_COLLECTION_FORMAT_SOURCE, outputs.lint_output, + lambda x: + "Tool data output reverse should use either format_source or format/ext" in x.warn_messages + and len(x.warn_messages) == 1 and len(x.error_messages) == 0 + ), + ( + OUTPUTS_DISCOVER_TOOL_PROVIDED_METADATA, outputs.lint_output, + lambda x: + len(x.warn_messages) == 0 and len(x.error_messages) == 0 + ), +] + +TEST_IDS = [ + 'outputs collection static elements with format_source', + 'outputs discover datatsets with tool provided metadata' +] + + +@pytest.mark.parametrize('tool_xml,lint_func,assert_func', TESTS, ids=TEST_IDS) +def test_tool_xml(tool_xml, lint_func, assert_func): + lint_ctx = LintContext('all') + tree = etree.ElementTree(element=etree.fromstring(tool_xml)) + lint_ctx.lint(name="test_lint", lint_func=lint_func, lint_target=tree) + assert assert_func(lint_ctx), ( + f"Warnings: {lint_ctx.warn_messages}\n" + f"Errors: {lint_ctx.error_messages}" + ) From 886f4ab42f55be0b34815d15425d8627a37bf2ea Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 21 Sep 2021 16:30:36 +0200 Subject: [PATCH 075/221] Drop more debug statements --- lib/galaxy/tools/parameters/validation.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/galaxy/tools/parameters/validation.py b/lib/galaxy/tools/parameters/validation.py index 103abe62cbd..9f495347190 100644 --- a/lib/galaxy/tools/parameters/validation.py +++ b/lib/galaxy/tools/parameters/validation.py @@ -193,11 +193,9 @@ class ExpressionValidator(Validator): self.expression = compile(expression, '', 'eval') def validate(self, value, trans=None): - log.error(f"ExpressionValidator.validate value {value} expression {self.expression}") try: evalresult = eval(self.expression, dict(value=value)) except Exception: - log.debug(f"Validator '{self.expression}' could not be evaluated on '{str(value)}'", exc_info=True) super().validate(False, value, f"Validator '{self.expression}' could not be evaluated on '%s'") super().validate(evalresult, value_to_show=value) From 40b70a58efba79d4f795b332b4dd3516e9461d4e Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 21 Sep 2021 17:43:43 +0200 Subject: [PATCH 076/221] Enable data_source tools with requirements This means the command section of the data_source tool can't use tools/data_source/data_source.py, but you can write data source tools that don't need this, like the ncbi_datasets_source tool. --- lib/galaxy/tools/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index ff97e25d9ac..13a8ba29604 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -709,12 +709,15 @@ class Tool(Dictifiable): # seem to require Galaxy's Python. # FIXME: the (instantiated) tool class should emit this behavior, and not # use inspection by string check - if self.tool_type not in ["default", "manage_data", "interactive"]: + if self.tool_type not in ["default", "manage_data", "interactive", "data_source"]: return True if self.tool_type == "manage_data" and self.profile < 18.09: return True + if self.tool_type == "data_source" and self.profile < 21.09: + return True + config = self.app.config preserve_python_environment = config.preserve_python_environment if preserve_python_environment == "always": From 6115d1c0f5bb19d1447d97baba76391751a4f1c2 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 21 Sep 2021 14:34:33 -0400 Subject: [PATCH 077/221] Drop Dataset.genome_index_tool_data relationship --- lib/galaxy/model/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 8133adeb788..97533462a4d 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2202,7 +2202,7 @@ class GenomeIndexToolData(Base, RepresentById): # TODO: params arg is lost indexer = Column(String(64)) user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) job = relationship('Job', back_populates='job') - dataset = relationship('Dataset', back_populates='genome_index_tool_data') + dataset = relationship('Dataset') user = relationship('User') @@ -8734,7 +8734,6 @@ mapper_registry.map_imperatively( hashes=relationship(DatasetHash, back_populates='dataset'), sources=relationship(DatasetSource, back_populates='dataset'), job_export_history_archive=relationship(JobExportHistoryArchive, back_populates='dataset'), - genome_index_tool_data=relationship(GenomeIndexToolData, back_populates='dataset'), history_associations=relationship(HistoryDatasetAssociation, back_populates='dataset'), library_associations=relationship(LibraryDatasetDatasetAssociation, primaryjoin=(LibraryDatasetDatasetAssociation.table.c.dataset_id == Dataset.table.c.id), From 2e113678571be20d29f3581cf63dc3a7196528d5 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 21 Sep 2021 14:38:11 -0400 Subject: [PATCH 078/221] Drop Dataset.job_export_history_archive --- lib/galaxy/model/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 97533462a4d..4483f9076ce 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2045,7 +2045,7 @@ class JobExportHistoryArchive(Base, RepresentById): compressed = Column(Boolean, index=True, default=False) history_attrs_filename = Column(TEXT) job = relationship('Job') - dataset = relationship('Dataset', back_populates='job_export_history_archive') + dataset = relationship('Dataset') history = relationship('History', back_populates='exports') ATTRS_FILENAME_HISTORY = 'history_attrs.txt' @@ -8733,7 +8733,6 @@ mapper_registry.map_imperatively( viewonly=True), hashes=relationship(DatasetHash, back_populates='dataset'), sources=relationship(DatasetSource, back_populates='dataset'), - job_export_history_archive=relationship(JobExportHistoryArchive, back_populates='dataset'), history_associations=relationship(HistoryDatasetAssociation, back_populates='dataset'), library_associations=relationship(LibraryDatasetDatasetAssociation, primaryjoin=(LibraryDatasetDatasetAssociation.table.c.dataset_id == Dataset.table.c.id), From f4d3b610fc097e333af99139f60edc931c931b95 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 21 Sep 2021 20:34:47 +0200 Subject: [PATCH 079/221] Restore truthy/falsy ExpressionValidator behavior Fixes https://github.com/galaxyproject/tools-iuc/issues/3974#issuecomment-924228314, broken in https://github.com/galaxyproject/galaxy/commit/a0046e43168c238c9b87b2a996e1f8c06503131a --- lib/galaxy/tools/parameters/validation.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tools/parameters/validation.py b/lib/galaxy/tools/parameters/validation.py index 3b5e322e0d9..159f2f42ad1 100644 --- a/lib/galaxy/tools/parameters/validation.py +++ b/lib/galaxy/tools/parameters/validation.py @@ -179,6 +179,16 @@ class ExpressionValidator(Validator): ... ValueError: Not gonna happen >>> t = p.validate("Fop") + >>> p = ToolParameter.build(None, XML(''' + ... + ... value + ... + ... ''')) + >>> p.validate("Foo") + >>> p.validate("") + Traceback (most recent call last): + ... + ValueError: Not gonna happen """ @classmethod @@ -200,7 +210,7 @@ class ExpressionValidator(Validator): except Exception: log.debug(f"Validator '{self.expression}' could not be evaluated on '{str(value)}'", exc_info=True) super().validate(False, value, f"Validator '{self.expression}' could not be evaluated on '%s'") - super().validate(evalresult, value_to_show=value) + super().validate(bool(evalresult), value_to_show=value) class InRangeValidator(ExpressionValidator): From 3ca8aa2742169e7e4b52bfbd59cd921ad9a43d93 Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Tue, 21 Sep 2021 14:46:43 -0400 Subject: [PATCH 080/221] Revised download() so parallel get() is utilized. Small refactor. --- lib/galaxy/objectstore/irods.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/objectstore/irods.py b/lib/galaxy/objectstore/irods.py index 9c504eafd82..9df1950809a 100644 --- a/lib/galaxy/objectstore/irods.py +++ b/lib/galaxy/objectstore/irods.py @@ -350,27 +350,16 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin): data_obj = None try: - data_obj = self.session.data_objects.get(data_object_path) + cache_path = self._get_cache_path(rel_path) + self.session.data_objects.get(data_object_path, cache_path) + log.debug("Pulled data object '%s' into cache to %s", rel_path, cache_path) + return True except (DataObjectDoesNotExist, CollectionDoesNotExist): log.warning("Collection or data object (%s) does not exist", data_object_path) return False finally: log.debug("irods_pt _download: %s", ipt_timer) - if self.cache_size > 0 and data_obj.__sizeof__() > self.cache_size: - log.critical("File %s is larger (%s) than the cache size (%s). Cannot download.", - rel_path, data_obj.__sizeof__(), self.cache_size) - log.debug("irods_pt _download: %s", ipt_timer) - return False - - log.debug("Pulled data object '%s' into cache to %s", rel_path, self._get_cache_path(rel_path)) - - with data_obj.open('r') as data_obj_fp, open(self._get_cache_path(rel_path), "wb") as cache_fp: - for chunk in iter(partial(data_obj_fp.read, CHUNK_SIZE), b''): - cache_fp.write(chunk) - log.debug("irods_pt _download: %s", ipt_timer) - return True - def _push_to_irods(self, rel_path, source_file=None, from_string=None): """ Push the file pointed to by ``rel_path`` to the iRODS. Extract folder name @@ -406,10 +395,10 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin): # Create sub-collection first self.session.collections.create(collection_path, recurse=True) - # Create data object - data_obj = self.session.data_objects.create(data_object_path, self.resource, **options) - if from_string: + # Create data object + data_obj = self.session.data_objects.create(data_object_path, self.resource, **options) + # Save 'from_string' as a file with data_obj.open('w') as data_obj_fp: data_obj_fp.write(from_string) From 0bb8f7faad74b95ce1a58ed83cec766243144acc Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 21 Sep 2021 14:48:02 -0400 Subject: [PATCH 081/221] Drop LDDA.info_association relationship --- lib/galaxy/model/__init__.py | 11 +---------- test/unit/model/test_mapping.py | 16 ---------------- 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 4483f9076ce..58f437b3cc6 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4940,9 +4940,7 @@ class LibraryDatasetDatasetInfoAssociation(Base, RepresentById): (LibraryDatasetDatasetInfoAssociation.library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.id) & (not_(LibraryDatasetDatasetInfoAssociation.deleted)) - ), - back_populates="info_association") - + )) template = relationship('FormDefinition', primaryjoin=(lambda: LibraryDatasetDatasetInfoAssociation.form_definition_id == FormDefinition.id)) # type: ignore @@ -8851,13 +8849,6 @@ mapper_registry.map_imperatively( primaryjoin=(HistoryDatasetAssociation.table.c.id == LibraryDatasetDatasetAssociation.table.c.copied_from_history_dataset_association_id), back_populates='copied_to_library_dataset_dataset_associations'), - info_association=relationship(LibraryDatasetDatasetInfoAssociation, - primaryjoin=(lambda: - (LibraryDatasetDatasetInfoAssociation.library_dataset_dataset_association_id - == LibraryDatasetDatasetAssociation.id) - & (not_(LibraryDatasetDatasetInfoAssociation.deleted)) - ), - back_populates='library_dataset_dataset_association'), ) ) diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 492a356a9d9..c87e0812100 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -3365,7 +3365,6 @@ class TestLibraryDatasetDatasetAssociation(BaseTest): implicitly_converted_dataset_association_factory, job_to_input_library_dataset_association, job_to_output_library_dataset_association, - library_dataset_dataset_info_association, ): copied_from_ldda = library_dataset_dataset_association_factory() copied_to_ldda = library_dataset_dataset_association_factory() @@ -3390,7 +3389,6 @@ class TestLibraryDatasetDatasetAssociation(BaseTest): obj.implicitly_converted_parent_datasets.append(icpda) obj.dependent_jobs.append(job_to_input_library_dataset_association) obj.creating_job_associations.append(job_to_output_library_dataset_association) - obj.info_association.append(library_dataset_dataset_info_association) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) @@ -3411,7 +3409,6 @@ class TestLibraryDatasetDatasetAssociation(BaseTest): assert stored_obj.dependent_jobs == [job_to_input_library_dataset_association] assert (stored_obj.creating_job_associations == [job_to_output_library_dataset_association]) - assert stored_obj.info_association == [library_dataset_dataset_info_association] delete_from_database(session, persisted) @@ -6887,19 +6884,6 @@ def library_dataset_dataset_association_tag_association(model, session): yield from dbcleanup_wrapper(session, instance) -@pytest.fixture -def library_dataset_dataset_info_association( - model, - session, - library_dataset_dataset_association, - form_definition, - form_values -): - instance = model.LibraryDatasetDatasetInfoAssociation( - library_dataset_dataset_association, form_definition, form_values) - yield from dbcleanup_wrapper(session, instance) - - @pytest.fixture def library_dataset_permission(model, session, library_dataset, role): instance = model.LibraryDatasetPermissions('a', library_dataset, role) From fb889d85a1fc96f97fa6e5ecd9e120be9b567135 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 21 Sep 2021 15:20:46 -0400 Subject: [PATCH 082/221] Guard against instance of GenomeIndexToolData --- lib/galaxy/tools/evaluation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index 5e2e748155c..10b5fcb6bf8 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -85,7 +85,7 @@ class ToolEvaluator: if get_special: special = get_special() if special: - out_data["output_file"] = special.fda + out_data["output_file"] = getattr(special, 'fda', None) # These can be passed on the command line if wanted as $__user_*__ incoming.update(model.User.user_template_environment(job.history and job.history.user)) From 793a7d167c0c9d3992329901f2ff79e592402aa0 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 21 Sep 2021 15:25:48 -0400 Subject: [PATCH 083/221] Add support for caching the Singularity cached images directory. --- .../deps/container_resolvers/mulled.py | 123 +++++++++++++++--- 1 file changed, 107 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/tool_util/deps/container_resolvers/mulled.py b/lib/galaxy/tool_util/deps/container_resolvers/mulled.py index e2136dfb081..9bd139ff7c0 100644 --- a/lib/galaxy/tool_util/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tool_util/deps/container_resolvers/mulled.py @@ -3,6 +3,10 @@ import logging import os import subprocess +from abc import ( + ABCMeta, + abstractmethod, +) from typing import NamedTuple, Optional from galaxy.util import ( @@ -75,6 +79,75 @@ class CachedV2MulledImageMultiTarget(NamedTuple): return image_name.rsplit("/")[-1] +class CacheDirectory(metaclass=ABCMeta): + def __init__(self, path, hash_func="v2"): + self.path = path + self.hash_func = hash_func + + def _list_cached_mulled_images_from_path(self): + contents = os.listdir(self.path) + sorted_images = version_sorted(contents) + raw_images = map(lambda name: identifier_to_cached_target(name, self.hash_func), sorted_images) + return list([i for i in raw_images if i is not None]) + + @abstractmethod + def list_cached_mulled_images_from_path(self): + """Generate a list of cached, mulled images in the cache.""" + + @abstractmethod + def invalidate_cache(self): + """Invalidate the cache.""" + + +class UncachedCacheDirectory(CacheDirectory): + cacher_type = "uncached" + + def list_cached_mulled_images_from_path(self): + return self._list_cached_mulled_images_from_path() + + def invalidate_cache(self): + pass + + +class DirMtimeCacheDirectory(CacheDirectory): + cacher_type = "dir_mtime" + + def __init__(self, path, **kwargs): + super().__init__(path, **kwargs) + self.invalidate_cache() + + def __get_mtime(self): + return os.stat(self.path).st_mtime + + def __cache(self): + self.__contents = self._list_cached_mulled_images_from_path() + self.__mtime = self.__get_mtime() + log.debug(f"Cached images in path {self.path} at directory mtime {self.__mtime}") + + def list_cached_mulled_images_from_path(self): + mtime = self.__get_mtime() + if mtime != self.__mtime: + if mtime < self.__mtime: + log.warning(f"Modification time '{mtime}' of cache directory '{self.path}' is older than previous " + f"modification time '{self.__mtime}'! Cache directory will be recached") + self.__cache() + return self.__contents + + def invalidate_cache(self): + self.__mtime = -1 + self.__contents = [] + + +def get_cache_directory_cacher(cacher_type): + # these can become a separate module and use plugin_config if we need more + cachers = { + UncachedCacheDirectory.cacher_type: UncachedCacheDirectory, + DirMtimeCacheDirectory.cacher_type: DirMtimeCacheDirectory, + } + cacher_type = cacher_type or "uncached" + return cachers[cacher_type] + + def list_docker_cached_mulled_images(namespace=None, hash_func="v2", resolution_cache=None): cache_key = "galaxy.tool_util.deps.container_resolvers.mulled:cached_images" if resolution_cache is not None and cache_key in resolution_cache: @@ -148,13 +221,6 @@ def identifier_to_cached_target(identifier, hash_func, namespace=None): return image -def list_cached_mulled_images_from_path(directory, hash_func="v2"): - contents = os.listdir(directory) - sorted_images = version_sorted(contents) - raw_images = map(lambda name: identifier_to_cached_target(name, hash_func), sorted_images) - return [i for i in raw_images if i is not None] - - def get_filter(namespace): prefix = "quay.io/" if namespace is None else f"quay.io/{namespace}" return lambda name: name.startswith(prefix) and name.count("/") == 2 @@ -231,16 +297,16 @@ def singularity_cached_container_description(targets, cache_directory, hash_func if len(targets) == 0: return None - if not os.path.exists(cache_directory): + if not os.path.exists(cache_directory.path): return None - cached_images = list_cached_mulled_images_from_path(cache_directory, hash_func=hash_func) + cached_images = cache_directory.list_cached_mulled_images_from_path() image = find_best_matching_cached_image(targets, cached_images, hash_func) container = None if image: container = ContainerDescription( - os.path.abspath(os.path.join(cache_directory, image.image_identifier)), + os.path.abspath(os.path.join(cache_directory.path, image.image_identifier)), type="singularity", shell=shell, ) @@ -363,8 +429,15 @@ class SingularityCliContainerResolver(CliContainerResolver): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.cache_directory = kwargs.get("cache_directory", os.path.join(kwargs['app_info'].container_image_cache_path, "singularity", "mulled")) - safe_makedirs(self.cache_directory) + self.cache_directory_path = kwargs.get("cache_directory", os.path.join(kwargs['app_info'].container_image_cache_path, "singularity", "mulled")) + self.cache_directory_cacher_type = kwargs.get("cache_directory_cacher_type", None) + self.cache_directory = None + self.hash_func = None + + def _init_cache_directory(self): + cacher_class = get_cache_directory_cacher(self.cache_directory_cacher_type) + self.cache_directory = cacher_class(self.cache_directory_path, hash_func=self.hash_func) + safe_makedirs(self.cache_directory.path) class CachedMulledDockerContainerResolver(CliContainerResolver): @@ -382,6 +455,7 @@ class CachedMulledDockerContainerResolver(CliContainerResolver): return None targets = mulled_targets(tool_info) + log.debug(f"Image name for tool {tool_info.tool_id}: {image_name(targets, self.hash_func)}") resolution_cache = kwds.get("resolution_cache") return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func, shell=self.shell, resolution_cache=resolution_cache) @@ -397,16 +471,18 @@ class CachedMulledSingularityContainerResolver(SingularityCliContainerResolver): def __init__(self, app_info=None, hash_func="v2", **kwds): super().__init__(app_info=app_info, **kwds) self.hash_func = hash_func + self._init_cache_directory() def resolve(self, enabled_container_types, tool_info, **kwds): if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: return None targets = mulled_targets(tool_info) + log.debug(f"Image name for tool {tool_info.tool_id}: {image_name(targets, self.hash_func)}") return singularity_cached_container_description(targets, self.cache_directory, hash_func=self.hash_func, shell=self.shell) def __str__(self): - return f"CachedMulledSingularityContainerResolver[cache_directory={self.cache_directory}]" + return f"CachedMulledSingularityContainerResolver[cache_directory={self.cache_directory.path}]" class MulledDockerContainerResolver(CliContainerResolver): @@ -446,6 +522,7 @@ class MulledDockerContainerResolver(CliContainerResolver): return None targets = mulled_targets(tool_info) + log.debug(f"Image name for tool {tool_info.tool_id}: {image_name(targets, self.hash_func)}") if len(targets) == 0: return None @@ -499,6 +576,7 @@ class MulledSingularityContainerResolver(SingularityCliContainerResolver, Mulled super().__init__(app_info=app_info, **kwds) self.namespace = namespace self.hash_func = hash_func + self._init_cache_directory() self.auto_install = string_as_bool(auto_install) def cached_container_description(self, targets, namespace, hash_func, resolution_cache): @@ -513,8 +591,9 @@ class MulledSingularityContainerResolver(SingularityCliContainerResolver, Mulled def pull(self, container): if self.cli_available: - cmds = container.build_mulled_singularity_pull_command(cache_directory=self.cache_directory, namespace=self.namespace) + cmds = container.build_mulled_singularity_pull_command(cache_directory=self.cache_directory.path, namespace=self.namespace) shell(cmds=cmds) + self.cache_directory.invalidate_cache() def __str__(self): return f"MulledSingularityContainerResolver[namespace={self.namespace}]" @@ -548,6 +627,7 @@ class BuildMulledDockerContainerResolver(CliContainerResolver): return None targets = mulled_targets(tool_info) + log.debug(f"Image name for tool {tool_info.tool_id}: {image_name(targets, self.hash_func)}") if len(targets) == 0: return None if self.auto_install or install: @@ -580,13 +660,14 @@ class BuildMulledSingularityContainerResolver(SingularityCliContainerResolver): 'involucro_bin': self._get_config_option("involucro_path", None) } self.hash_func = hash_func + self._init_cache_directory() self.auto_install = string_as_bool(auto_install) self._mulled_kwds = { 'channels': self._get_config_option("mulled_channels", DEFAULT_CHANNELS), 'hash_func': self.hash_func, 'command': 'build-and-test', 'singularity': True, - 'singularity_image_dir': self.cache_directory, + 'singularity_image_dir': self.cache_directory.path, } self.auto_init = self._get_config_option("involucro_auto_init", True) @@ -595,6 +676,7 @@ class BuildMulledSingularityContainerResolver(SingularityCliContainerResolver): return None targets = mulled_targets(tool_info) + log.debug(f"Image name for tool {tool_info.tool_id}: {image_name(targets, self.hash_func)}") if len(targets) == 0: return None @@ -612,13 +694,22 @@ class BuildMulledSingularityContainerResolver(SingularityCliContainerResolver): return involucro_context def __str__(self): - return f"BuildSingularityContainerResolver[cache_directory={self.cache_directory}]" + return f"BuildSingularityContainerResolver[cache_directory={self.cache_directory.path}]" def mulled_targets(tool_info): return requirements_to_mulled_targets(tool_info.requirements) +def image_name(targets, hash_func): + if len(targets) == 0: + return "no targets" + elif hash_func == "v2": + return v2_image_name(targets) + else: + return v1_image_name(targets) + + __all__ = ( "CachedMulledDockerContainerResolver", "CachedMulledSingularityContainerResolver", From 98ad38059040643e415dac3985a51c9263374382 Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Tue, 21 Sep 2021 16:28:51 -0400 Subject: [PATCH 084/221] Fixed some lint issues --- lib/galaxy/objectstore/irods.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/galaxy/objectstore/irods.py b/lib/galaxy/objectstore/irods.py index 9df1950809a..6c28cf79436 100644 --- a/lib/galaxy/objectstore/irods.py +++ b/lib/galaxy/objectstore/irods.py @@ -5,7 +5,6 @@ import logging import os import shutil from datetime import datetime -from functools import partial from pathlib import Path try: @@ -347,7 +346,6 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin): collection_path = f"{self.home}/{str(subcollection_name)}" data_object_path = f"{collection_path}/{str(data_object_name)}" - data_obj = None try: cache_path = self._get_cache_path(rel_path) From 9bd37cac4bb63e9e1a2ae1652c14feffb5bdde57 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 21 Sep 2021 15:42:54 -0400 Subject: [PATCH 085/221] Drop JobStateHistory.job relationship Even though it seems we'd need this, we don't: we store the job.id in the job_state_history table as a foreign key, but we never refer to jsh.job, and, I think, we should never need to. --- lib/galaxy/model/__init__.py | 5 ++--- test/unit/model/test_mapping.py | 7 ------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 58f437b3cc6..aece73eceb3 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -916,7 +916,7 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): lazy=True, back_populates='job') tasks = relationship('Task', back_populates='job') output_datasets = relationship('JobToOutputDatasetAssociation', back_populates='job') - state_history = relationship('JobStateHistory', back_populates='job') + state_history = relationship('JobStateHistory') text_metrics = relationship('JobMetricText') numeric_metrics = relationship('JobMetricNumeric') job = relationship('GenomeIndexToolData', back_populates='job') @@ -1875,10 +1875,9 @@ class JobStateHistory(Base, RepresentById): job_id = Column(Integer, ForeignKey('job.id'), index=True) state = Column(String(64), index=True) info = Column(TrimmedString(255)) - job = relationship('Job', back_populates='state_history') def __init__(self, job): - self.job = job + self.job_id = job.id self.state = job.state self.info = job.info diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index c87e0812100..079fb8e783a 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -2780,13 +2780,6 @@ class TestJobStateHistory(BaseTest): assert stored_obj.state == state assert stored_obj.info == info - def test_relationships(self, session, cls_, job): - obj = cls_(job) - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.job.id == job.id - class TestJobToImplicitOutputDatasetCollectionAssociation(BaseTest): From e21fff988990f08f2ede73cc2152e900c1cd4544 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 21 Sep 2021 18:14:43 -0400 Subject: [PATCH 086/221] Fix tagging model tests We don't need the backref attributes: we can simply use a different query to get the same data --- test/unit/data/test_galaxy_mapping.py | 83 +++++++++++++-------------- 1 file changed, 41 insertions(+), 42 deletions(-) diff --git a/test/unit/data/test_galaxy_mapping.py b/test/unit/data/test_galaxy_mapping.py index 3d0f273d9f8..2d56ccf1137 100644 --- a/test/unit/data/test_galaxy_mapping.py +++ b/test/unit/data/test_galaxy_mapping.py @@ -6,7 +6,7 @@ import uuid from tempfile import NamedTemporaryFile import pytest -from sqlalchemy import inspect +from sqlalchemy import inspect, select import galaxy.datatypes.registry import galaxy.model @@ -234,47 +234,46 @@ class MappingTests(BaseModelTestCase): assert new_ldda.library_dataset.expired_datasets[0] == ldda assert target_folder.item_count == 1 -# def test_tags(self): -# model = self.model -# -# my_tag = model.Tag(name="Test Tag") -# u = model.User(email="tagger@example.com", password="password") -# self.persist(my_tag, u) -# -# def tag_and_test(taggable_object, tag_association_class, backref_name): -# assert len(getattr(self.query(model.Tag).filter(model.Tag.name == "Test Tag").all()[0], backref_name)) == 0 -# -# tag_association = tag_association_class() -# tag_association.tag = my_tag -# taggable_object.tags = [tag_association] -# self.persist(tag_association, taggable_object) -# -# assert len(getattr(self.query(model.Tag).filter(model.Tag.name == "Test Tag").all()[0], backref_name)) == 1 -# -# sw = model.StoredWorkflow() -# sw.user = u -# tag_and_test(sw, model.StoredWorkflowTagAssociation, "tagged_stored_workflows") -# -# h = model.History(name="History for Tagging", user=u) -# tag_and_test(h, model.HistoryTagAssociation, "tagged_histories") -# -# d1 = model.HistoryDatasetAssociation(extension="txt", history=h, create_dataset=True, sa_session=model.session) -# tag_and_test(d1, model.HistoryDatasetAssociationTagAssociation, "tagged_history_dataset_associations") -# -# page = model.Page() -# page.user = u -# tag_and_test(page, model.PageTagAssociation, "tagged_pages") -# -# visualization = model.Visualization() -# visualization.user = u -# tag_and_test(visualization, model.VisualizationTagAssociation, "tagged_visualizations") -# -# dataset_collection = model.DatasetCollection(collection_type="paired") -# history_dataset_collection = model.HistoryDatasetCollectionAssociation(collection=dataset_collection) -# tag_and_test(history_dataset_collection, model.HistoryDatasetCollectionTagAssociation, "tagged_history_dataset_collections") -# -# library_dataset_collection = model.LibraryDatasetCollectionAssociation(collection=dataset_collection) -# tag_and_test(library_dataset_collection, model.LibraryDatasetCollectionTagAssociation, "tagged_library_dataset_collections") + def test_tags(self): + model = self.model + TAG_NAME = 'Test Tag' + my_tag = model.Tag(name=TAG_NAME) + u = model.User(email="tagger@example.com", password="password") + self.persist(my_tag, u) + + def tag_and_test(taggable_object, tag_association_class): + q = select(tag_association_class).join(model.Tag).where(model.Tag.name == TAG_NAME) + + assert len(model.session.execute(q).all()) == 0 + + tag_association = tag_association_class() + tag_association.tag = my_tag + taggable_object.tags = [tag_association] + self.persist(tag_association, taggable_object) + + assert len(model.session.execute(q).all()) == 1 + + sw = model.StoredWorkflow(user=u) + tag_and_test(sw, model.StoredWorkflowTagAssociation) + + h = model.History(name="History for Tagging", user=u) + tag_and_test(h, model.HistoryTagAssociation) + + d1 = model.HistoryDatasetAssociation(extension="txt", history=h, create_dataset=True, sa_session=model.session) + tag_and_test(d1, model.HistoryDatasetAssociationTagAssociation) + + page = model.Page(user=u) + tag_and_test(page, model.PageTagAssociation) + + visualization = model.Visualization(user=u) + tag_and_test(visualization, model.VisualizationTagAssociation) + + dataset_collection = model.DatasetCollection(collection_type="paired") + history_dataset_collection = model.HistoryDatasetCollectionAssociation(collection=dataset_collection) + tag_and_test(history_dataset_collection, model.HistoryDatasetCollectionTagAssociation) + + library_dataset_collection = model.LibraryDatasetCollectionAssociation(collection=dataset_collection) + tag_and_test(library_dataset_collection, model.LibraryDatasetCollectionTagAssociation) def test_collection_get_interface(self): model = self.model From a5c9ee19f4e53120c4564697a8563cd381e57897 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 22 Sep 2021 12:25:32 +0200 Subject: [PATCH 087/221] restructure linter tests ie split in multiple files --- test/unit/tool_util/test_tool_linters.py | 30 ++++++++++++++++++ .../tool_util/test_tool_linters_general.py | 31 ++----------------- ...s_input.py => test_tool_linters_inputs.py} | 18 +---------- .../tool_util/test_tool_linters_outputs.py | 16 ---------- 4 files changed, 33 insertions(+), 62 deletions(-) create mode 100644 test/unit/tool_util/test_tool_linters.py rename test/unit/tool_util/{test_tool_linters_input.py => test_tool_linters_inputs.py} (93%) diff --git a/test/unit/tool_util/test_tool_linters.py b/test/unit/tool_util/test_tool_linters.py new file mode 100644 index 00000000000..8723f097501 --- /dev/null +++ b/test/unit/tool_util/test_tool_linters.py @@ -0,0 +1,30 @@ +import pytest + +from galaxy.tool_util.lint import LintContext +from galaxy.tool_util.parser.xml import XmlToolSource +from galaxy.util import etree +from galaxy.util.getargspec import getfullargspec +from . import ( + test_tool_linters_general, + test_tool_linters_inputs, + test_tool_linters_outputs +) + +TESTS = test_tool_linters_general.TESTS + test_tool_linters_inputs.TESTS + test_tool_linters_outputs.TESTS +TEST_IDS = test_tool_linters_general.TEST_IDS + test_tool_linters_inputs.TEST_IDS + test_tool_linters_outputs.TEST_IDS + + +@pytest.mark.parametrize('tool_xml,lint_func,assert_func', TESTS, ids=TEST_IDS) +def test_tool_xml(tool_xml, lint_func, assert_func): + lint_ctx = LintContext('all') + # the general linter gets XMLToolSource and all others + # an ElementTree + first_arg = getfullargspec(lint_func).args[0] + lint_target = etree.ElementTree(element=etree.fromstring(tool_xml)) + if first_arg != "tool_xml": + lint_target = XmlToolSource(lint_target) + lint_ctx.lint(name="test_lint", lint_func=lint_func, lint_target=lint_target) + assert assert_func(lint_ctx), ( + f"Warnings: {lint_ctx.warn_messages}\n" + f"Errors: {lint_ctx.error_messages}" + ) diff --git a/test/unit/tool_util/test_tool_linters_general.py b/test/unit/tool_util/test_tool_linters_general.py index 7d5284b625b..65aeb2d53e9 100644 --- a/test/unit/tool_util/test_tool_linters_general.py +++ b/test/unit/tool_util/test_tool_linters_general.py @@ -1,20 +1,4 @@ -import pytest - -from galaxy.tool_util.lint import LintContext from galaxy.tool_util.linters import general -from galaxy.tool_util.parser.xml import XmlToolSource -from galaxy.util import etree - - -def lint_general(xml_tree, lint_ctx): - """Wrap calling of lint_general to provide XmlToolSource argument. - - This allows general.lint_general to be called with the other linters which - take an XmlTree as an argument. - """ - tool_source = XmlToolSource(xml_tree) - return general.lint_general(tool_source, lint_ctx) - WHITESPACE_IN_VERSIONS_AND_NAMES = """